commit 75d42c4597abe3b847a49b5986f682f327803bb0 Author: April Ayres-Griffiths Date: Sat Jan 20 11:05:04 2018 +1100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a1338d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9cecc1d --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..31e92c9 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +DSKalyzer is a cross-platform command-line tool for manipulating and managing Apple II DSK (and other) images. + +Download from: https://github.com/paleotronic/dskalyzer/releases + +Features include: + +- Read from ProDOS, DOS 3.X, RDOS and Pascal disk images; +- ProDOS or DOS ordered; 2MG and NIB; 113-800K +- Write to Prodos and DOS 3.3 disk images; +- Extract and convert binary, text and detokenize BASIC files (Integer and Applesoft); +- Write binary, text and retokenized BASIC (Applesoft) files back to disk images; +- Copy and move files between disk images; delete files, create new folders (ProDOS), etc; +- Generate disk reports that provide track and sector information, text extraction and more; +- Compare multiple disks to determine duplication, or search disks for text or filenames. +- Use command-line flags (allows for automation) or an interactive shell; +- Builds for MacOS, Windows, Linux, FreeBSD and Raspberry Pi. +- Open source; GPLv3 licensed. +- Written in Go! + +DSKalyzer is a command line tool for analyzing and managing Apple II DSK images and their archives. Its features include not only the standard set of disk manipulation tools -- extract (with text conversion), import to disk (including tokenisation of Applesoft BASIC), delete, and so forth -- but also the ability to identify duplicates — complete, active sector, and subset; find file, sector and other commonalities between disks (including as a percentage of similarity or difference); search de-tokenized BASIC, text and binary / sector data; generate reports identifying and / or collating disk type, DOS, geometry, size, and so forth; allowing for easier, semi-automated DSK archival management and research. + +DSKalyzer works by first “ingesting” your disk(s), creating an index containing various pieces of information (disk / sector / file hashes, catalogs, text data, binary data etc.) about each disk that is then searchable using the same tool. This way you can easily find information stored on disks without tediously searching manually or through time-consuming multiple image scans. You can also identify duplicates, quasi-duplicates (disks with only minor differences or extraneous data), or iterations, reducing redundancies. + +Once you've identified a search you can also extract selected files. DSKalyzer can report to standard output (terminal), to a text file, or to a CSV file. +``` +Shell commands (executing DSKalyzer without flags enters shell): + +Note: You must mount a disk before performing tasks on it. + +analyze Process disk using dskalyzer analytics +cat Display file information +cd Change local path +copy Copy files from one volume to another +delete Remove file from disk +disks List mounted volumes +extract extract file from disk image +help Shows this help +info Information about the current disk +ingest Ingest directory containing disks (or single disk) into system +lock Lock file on the disk +ls List local files +mkdir Create a directory on disk +mount Mount a disk image +move Move files from one volume to another +put Copy local file to disk +quarantine Like report, but allow moving dupes to a backup folder +quit Leave this place +rename Rename a file on the disk +report Run a report +target Select mounted volume as default +unlock Unlock file on the disk +unmount unmount disk image + +Command-line flags: + +(Note: You must ingest your disk library before you can run comparison or search operations on it) + + -active-sector-partial + Run partial sector match (active only) against all disks + -active-sector-subset + Run subset (active) sector match against all disks + -adorned + Extract files named similar to CP (default true) + -all-file-partial + Run partial file match against all disks + -all-file-subset + Run subset file match against all disks + -all-sector-partial + Run partial sector match (all) against all disks + -all-sector-subset + Run subset (non-zero) sector match against all disks + -as-dupes + Run active sectors only disk dupe report + -as-partial + Run partial active sector match against single disk (-disk required) + -c Cache data to memory for quicker processing (default true) + -cat-dupes + Run duplicate catalog report + -catalog + List disk contents (-with-disk) + -csv + Output data to CSV format + -datastore string + Database of disk fingerprints for checking (default "/Users/melody/DSKalyzer/fingerprints") + -dir + Directory specified disk (needs -disk) + -dir-create string + Directory to create (-with-disk) + -dir-format string + Format of dir (default "{filename} {type} {size:kb} Checksum: {sha256}") + -extract string + Extract files/disks matched in searches ('#'=extract disk, '@'=extract files) + -file string + Search for other disks containing file + -file-delete string + File to delete (-with-disk) + -file-dupes + Run file dupe report + -file-extract string + File to delete from disk (-with-disk) + -file-partial + Run partial file match against single disk (-disk required) + -file-put string + File to put on disk (-with-disk) + -force + Force re-ingest disks that already exist + -ingest string + Disk file or path to ingest + -ingest-mode int + Ingest mode: + 0=Fingerprints only + 1=Fingerprints + text + 2=Fingerprints + sector data + 3=All (default 1) + -max-diff int + Maximum different # files for -all-file-partial + -min-same int + Minimum same # files for -all-file-partial + -out string + Output file (empty for stdout) + -quarantine + Run -as-dupes and -whole-disk in quarantine mode + -query string + Disk file to query or analyze + -search-filename string + Search database for file with name + -search-sha string + Search database for file with checksum + -search-text string + Search database for file containing text + -select + Select files for analysis or search based on file/dir/mask + -shell + Start interactive mode + -shell-batch string + Execute shell command(s) from file and exit + -similarity float + Object match threshold for -*-partial reports (default 0.9) + -verbose + Log to stderr + -whole-dupes + Run whole disk dupe report + -with-disk string + Perform disk operation (-file-extract,-file-put,-file-delete) +``` +Getting Started + +Ingest your disk collection, so dskalyzer can report on them: + +dskalyzer -ingest "C:\Users\myname\LotsOfDisks" +Simple Reports + +Find Whole Disk duplicates: + +dskalyzer -whole-dupes +Find Active Sectors duplicates (inactive sectors can be different): + +dskalyzer -as-dupes +Find Duplicate files across disks: + +dskalyzer -file-dupes +Limiting reports to subdirectories + +Find Active Sector duplicates but only under a folder: + +dskalyzer -as-dupes -select "C:\Users\myname\LotsOfDisks\Operating Systems" +``` diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..007d0dd --- /dev/null +++ b/USAGE.md @@ -0,0 +1,32 @@ +## Usage examples + +Ingest your disk collection, so dskalyzer can report on them: + +``` +dskalyzer -ingest C:\Users\myname\LotsOfDisks +``` + +Find Whole Disk duplicates: + +``` +dskalyzer -whole-dupes +``` + +Find Active Sectors duplicates (inactive sectors can be different): + +``` +dskalyzer -as-dupes +``` + +Find Duplicate files across disks: + +``` +dskalyzer -file-dupes +``` + +Find Active Sector duplicates but only under a folder: + +``` +dskalyzer -as-dupes -select "C:\Users\myname\LotsOfDisks\Operating Systems" +``` + diff --git a/banner.go b/banner.go new file mode 100644 index 0000000..b12daca --- /dev/null +++ b/banner.go @@ -0,0 +1,17 @@ +package main + +import ( + "encoding/base64" + "os" +) + +var text = `ICBfXyAgICAgICAgICAgX18gICAgICAgICAgICAgICAgX19fICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF9fICAgICANCiAvXCBcICAgICAgICAgL1wgXCAgICAgICAgICAgICAgL1xfIFwgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC9cIFwgICAgDQogXF9cIFwgICAgX19fX1wgXCBcLydcICAgICAgX18gIFwvL1wgXCAgICBfXyAgX18gIF9fX18gICAgICBfXyAgIF8gX19cIFwgXCAgIA0KIC8nX2AgXCAgLycsX19cXCBcICwgPCAgICAvJ19fYFwgIFwgXCBcICAvXCBcL1wgXC9cXyAsYFwgIC8nX19gXC9cYCdfX1wgXCBcICANCi9cIFxMXCBcL1xfXywgYFxcIFwgXFxgXCAvXCBcTFwuXF8gXF9cIFxfXCBcIFxfXCBcL18vICAvXy9cICBfXy9cIFwgXC8gXCBcX1wgDQpcIFxfX18sX1wvXF9fX18vIFwgXF9cIFxfXCBcX18vLlxfXC9cX19fX1xcL2BfX19fIFwvXF9fX19cIFxfX19fXFwgXF9cICBcL1xfXA0KIFwvX18sXyAvXC9fX18vICAgXC9fL1wvXy9cL19fL1wvXy9cL19fX18vIGAvX19fLz4gXC9fX19fL1wvX19fXy8gXC9fLyAgIFwvXy8NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAvXF9fXy8gICAgICAgICAgICAgICAgICAgICAgICAgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXC9fXy8gICAgICAgICAgICAgICAgICAgICAgICAgIA0K` + +func banner() { + + t, _ := base64.StdEncoding.DecodeString(text) + + os.Stderr.WriteString(string(t) + "\r\n") + os.Stderr.WriteString("(c) 2015 - 2017 Paleotronic.com\n\n") + +} diff --git a/data.go b/data.go new file mode 100644 index 0000000..6e76577 --- /dev/null +++ b/data.go @@ -0,0 +1,947 @@ +package main + +import ( + "errors" + "fmt" + "runtime" + "time" + + "crypto/md5" + "encoding/hex" + + "os" + + "strings" + + "encoding/gob" + + "path/filepath" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +type Disk struct { + FullPath string + Filename string + SHA256 string // Sha of whole disk + SHA256Active string // Sha of active sectors/blocks only + Format string + FormatID disk.DiskFormat + Bitmap []bool + Tracks, Sectors, Blocks int + Files DiskCatalog + ActiveSectors DiskSectors + //ActiveBlocks DiskBlocks + InactiveSectors DiskSectors + //InactiveBlocks DiskBlocks + MatchFactor float64 + MatchFiles map[*DiskFile]*DiskFile + MissingFiles, ExtraFiles []*DiskFile + IngestMode int + source string +} + +type ByMatchFactor []*Disk + +func (s ByMatchFactor) Len() int { + return len(s) +} + +func (s ByMatchFactor) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s ByMatchFactor) Less(i, j int) bool { + return s[i].MatchFactor < s[j].MatchFactor +} + +type TypeCode int + +const ( + TypeMask_AppleDOS TypeCode = 0x0000 + TypeMask_ProDOS TypeCode = 0x0100 + TypeMask_Pascal TypeCode = 0x0200 + TypeMask_RDOS TypeCode = 0x0300 +) + +type DiskFile struct { + Filename string + Type string + Ext string + TypeCode TypeCode + SHA256 string + Size int + LoadAddress int + Text []byte + Data []byte + Locked bool + Created time.Time + Modified time.Time +} + +func (d *DiskFile) GetNameAdorned() string { + + var ext string + switch d.TypeCode & 0xff00 { + case TypeMask_AppleDOS: + ext = disk.FileType(d.TypeCode & 0xff).Ext() + case TypeMask_ProDOS: + ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext() + case TypeMask_RDOS: + ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext() + case TypeMask_Pascal: + ext = disk.PascalFileType(d.TypeCode & 0xff).Ext() + } + + return fmt.Sprintf("%s#0x%.4x.%s", d.Filename, d.LoadAddress, ext) + +} + +func (d *DiskFile) GetName() string { + + var ext string + switch d.TypeCode & 0xff00 { + case TypeMask_AppleDOS: + ext = disk.FileType(d.TypeCode & 0xff).Ext() + case TypeMask_ProDOS: + ext = disk.ProDOSFileType(d.TypeCode & 0xff).Ext() + case TypeMask_RDOS: + ext = disk.RDOSFileType(d.TypeCode & 0xff).Ext() + case TypeMask_Pascal: + ext = disk.PascalFileType(d.TypeCode & 0xff).Ext() + } + + return fmt.Sprintf("%s.%s", d.Filename, ext) + +} + +type DiskCatalog []*DiskFile +type DiskSectors []*DiskSector +type DiskBlocks []*DiskBlock + +type DiskSector struct { + Track int + Sector int + + SHA256 string + + Data []byte +} + +type DiskBlock struct { + Block int + + SHA256 string +} + +func (i Disk) LogBitmap(id int) { + + l := loggy.Get(id) + + if i.Tracks > 0 { + + for t := 0; t < i.Tracks; t++ { + + line := fmt.Sprintf("Track %.2d: ", t) + + for s := 0; s < i.Sectors; s++ { + if i.Bitmap[t*i.Sectors+s] { + line += fmt.Sprintf("%.2x ", s) + } else { + line += ":: " + } + } + + l.Logf("%s", line) + } + + } else if i.Blocks > 0 { + + tr := i.Blocks / 16 + sc := 16 + + for t := 0; t < tr; t++ { + + line := fmt.Sprintf("Block %.2d: ", t) + + for s := 0; s < sc; s++ { + if i.Bitmap[t*i.Sectors+s] { + line += fmt.Sprintf("%.2x ", s) + } else { + line += ":: " + } + } + + l.Logf("%s", line) + } + + } + +} + +func (d Disk) GetFilename() string { + + sum := md5.Sum([]byte(d.Filename)) + + // fmt.Printf("checksum: [%s] -> [%s]\n", d.Filename, hex.EncodeToString(sum[:])) + + ff := fmt.Sprintf("%s/%d", strings.Trim(filepath.Dir(d.FullPath), "/"), d.FormatID.ID) + "_" + d.SHA256 + "_" + d.SHA256Active + "_" + hex.EncodeToString(sum[:]) + ".fgp" + + if runtime.GOOS == "windows" { + ff = strings.Replace(ff, ":", "", -1) + ff = strings.Replace(ff, "\\", "/", -1) + } + + return ff + +} + +func (d Disk) WriteToFile(filename string) error { + + // b, err := yaml.Marshal(d) + + // if err != nil { + // return err + // } + l := loggy.Get(0) + + _ = os.MkdirAll(filepath.Dir(filename), 0755) + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + enc := gob.NewEncoder(f) + enc.Encode(d) + + l.Logf("Created %s", filename) + + return nil +} + +func (d *Disk) ReadFromFile(filename string) error { + // b, err := ioutil.ReadFile(filename) + // if err != nil { + // return err + // } + // err = yaml.Unmarshal(b, d) + + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + + dec := gob.NewDecoder(f) + err = dec.Decode(d) + + d.source = filename + + return err +} + +// GetExactBinaryMatches returns disks with the same Global SHA256 +func (d *Disk) GetExactBinaryMatches(filter []string) []*Disk { + + l := loggy.Get(0) + + var out []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_"+d.SHA256+"_*_*.fgp") + if !exists { + return out + } + + for _, m := range matches { + l.Logf(":: Checking %s", m) + if item, err := cache.Get(m); err == nil { + if item.FullPath != d.FullPath { + out = append(out, item) + } + } + } + + return out +} + +// GetActiveSectorBinaryMatches returns disks with the same Active SHA256 +func (d *Disk) GetActiveSectorBinaryMatches(filter []string) []*Disk { + + l := loggy.Get(0) + + var out []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_"+d.SHA256Active+"_*.fgp") + if !exists { + return out + } + + for _, m := range matches { + l.Logf(":: Checking %s", m) + + if item, err := cache.Get(m); err == nil { + if item.FullPath != d.FullPath { + out = append(out, item) + } + } + } + + return out +} + +func (d *Disk) GetFileMap() map[string]*DiskFile { + + out := make(map[string]*DiskFile) + + for _, file := range d.Files { + f := file + out[file.SHA256] = f + } + + return out + +} + +func (d *Disk) GetUtilizationMap() map[string]string { + + out := make(map[string]string) + + if len(d.ActiveSectors) > 0 { + + for _, block := range d.ActiveSectors { + + key := fmt.Sprintf("T%dS%d", block.Track, block.Sector) + out[key] = block.SHA256 + + } + + } + + return out + +} + +// CompareChunks returns a value 0-1 +func (d *Disk) CompareChunks(b *Disk) (float64, float64, float64, float64) { + + l := loggy.Get(0) + + if d.FormatID != b.FormatID { + l.Logf("Trying to compare disks of different types") + return 0, 0, 0, 0 + } + + switch d.FormatID.ID { + case disk.DF_RDOS_3: + return d.compareSectorsPositional(b) + case disk.DF_RDOS_32: + return d.compareSectorsPositional(b) + case disk.DF_RDOS_33: + return d.compareSectorsPositional(b) + case disk.DF_PASCAL: + return d.compareBlocksPositional(b) + case disk.DF_DOS_SECTORS_13: + return d.compareSectorsPositional(b) + case disk.DF_DOS_SECTORS_16: + return d.compareSectorsPositional(b) + case disk.DF_PRODOS: + return d.compareBlocksPositional(b) + case disk.DF_PRODOS_800KB: + return d.compareBlocksPositional(b) + } + + return 0, 0, 0, 0 + +} + +func (d *Disk) compareSectorsPositional(b *Disk) (float64, float64, float64, float64) { + + l := loggy.Get(0) + + var sameSectors float64 + var diffSectors float64 + var dNotb float64 + var bNotd float64 + var emptySectors float64 + var dTotal, bTotal float64 + + var dmap = d.GetUtilizationMap() + var bmap = b.GetUtilizationMap() + + for t := 0; t < d.FormatID.TPD(); t++ { + + for s := 0; s < d.FormatID.SPT(); s++ { + + key := fmt.Sprintf("T%dS%d", t, s) + + dCk, dEx := dmap[key] + bCk, bEx := bmap[key] + + switch { + case dEx && bEx: + if dCk == bCk { + sameSectors += 1 + } else { + diffSectors += 1 + } + dTotal += 1 + bTotal += 1 + case dEx && !bEx: + dNotb += 1 + dTotal += 1 + case !dEx && bEx: + bNotd += 1 + bTotal += 1 + default: + emptySectors += 1 + } + + } + + } + + l.Debugf("Same Sectors : %f", sameSectors) + l.Debugf("Differing Sectors: %f", diffSectors) + l.Debugf("Not in other disk: %f", dNotb) + l.Debugf("Not in this disk : %f", bNotd) + + // return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal + return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal + +} + +func (d *Disk) compareBlocksPositional(b *Disk) (float64, float64, float64, float64) { + + l := loggy.Get(0) + + var sameSectors float64 + var diffSectors float64 + var dNotb float64 + var bNotd float64 + var emptySectors float64 + var dTotal, bTotal float64 + + var dmap = d.GetUtilizationMap() + var bmap = b.GetUtilizationMap() + + for t := 0; t < d.FormatID.BPD(); t++ { + + key := fmt.Sprintf("B%d", t) + + dCk, dEx := dmap[key] + bCk, bEx := bmap[key] + + switch { + case dEx && bEx: + if dCk == bCk { + sameSectors += 1 + } else { + diffSectors += 1 + } + dTotal += 1 + bTotal += 1 + case dEx && !bEx: + dNotb += 1 + dTotal += 1 + case !dEx && bEx: + bNotd += 1 + bTotal += 1 + default: + emptySectors += 1 + } + + } + + l.Debugf("Same Blocks : %f", sameSectors) + l.Debugf("Differing Blocks : %f", diffSectors) + l.Debugf("Not in other disk: %f", dNotb) + l.Debugf("Not in this disk : %f", bNotd) + + // return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal + return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / bTotal + +} + +// GetActiveSectorBinaryMatches returns disks with the same Active SHA256 +func (d *Disk) GetPartialMatches(filter []string) ([]*Disk, []*Disk, []*Disk) { + + l := loggy.Get(0) + + var superset []*Disk = make([]*Disk, 0) + var subset []*Disk = make([]*Disk, 0) + var identical []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp") + if !exists { + return superset, subset, identical + } + + for _, m := range matches { + //item := &Disk{} + if item, err := cache.Get(m); err == nil { + if item.FullPath != d.FullPath { + // only here if not looking at same disk + l.Logf(":: Checking overlapping data blocks %s", item.Filename) + l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + dSame, iSame, dDiff, iDiff := d.CompareChunks(item) + l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename) + l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename) + l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100) + l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100) + + if dSame == 1 && iSame < 1 { + superset = append(superset, item) + } else if iSame == 1 && dSame < 1 { + subset = append(subset, item) + } else if iSame == 1 && dSame == 1 { + identical = append(identical, item) + } + } + } + } + + return superset, subset, identical +} + +func (d *Disk) GetPartialMatchesWithThreshold(t float64, filter []string) []*Disk { + + l := loggy.Get(0) + + var matchlist []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, fmt.Sprintf("%d", d.FormatID)+"_*_*_*.fgp") + if !exists { + return matchlist + } + + var lastPc int = -1 + for i, m := range matches { + //item := &Disk{} + if item, err := cache.Get(m); err == nil { + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc)) + } + + if item.FullPath != d.FullPath { + // only here if not looking at same disk + l.Logf(":: Checking overlapping data blocks %s", item.Filename) + // l.Log("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + dSame, _, _, _ := d.CompareChunks(item) + // l.Logf("== This disk shares %.2f percent of its allocated blocks with %s", dSame*100, item.Filename) + // l.Logf("!= This disk differs %.2f percent of its allocate blocks with %s", dDiff*100, item.Filename) + // l.Logf("== %s shares %.2f of its blocks with this disk", item.Filename, iSame*100) + // l.Logf("!= %s differs %.2f of its blocks with this disk", item.Filename, iDiff*100) + + item.MatchFactor = dSame + + if dSame >= t { + matchlist = append(matchlist, item) + } + } + + fmt.Print("\r") + lastPc = pc + } + } + + return matchlist +} + +func Aggregate(f func(d *Disk, collector interface{}), collector interface{}, pathfilter []string) { + + l := loggy.Get(0) + + exists, matches := existsPattern(*baseName, pathfilter, "*_*_*_*.fgp") + if !exists { + return + } + + var lastPc int = -1 + for i, m := range matches { + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("\rAggregating data... %d%% ", pc)) + } + + l.Logf(":: Checking %s", m) + //item := &Disk{} + if item, err := cache.Get(m); err == nil { + f(item, collector) + } + + } + + os.Stderr.WriteString("Done.\n") + + return +} + +func (d *Disk) CompareFiles(b *Disk) float64 { + + var sameFiles float64 + var missingFiles float64 + var extraFiles float64 + + var dmap = d.GetFileMap() + var bmap = b.GetFileMap() + + for fileCk, info := range dmap { + + if info.Size == 0 { + continue + } + + binfo, bEx := bmap[fileCk] + + if bEx { + sameFiles += 1 + // file match + if b.MatchFiles == nil { + b.MatchFiles = make(map[*DiskFile]*DiskFile) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + b.MatchFiles[binfo] = info + } else { + missingFiles += 1 + // file match + if b.MissingFiles == nil { + b.MissingFiles = make([]*DiskFile, 0) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + b.MissingFiles = append(b.MissingFiles, info) + } + + } + + for fileCk, info := range bmap { + + if info.Size == 0 { + continue + } + + _, dEx := dmap[fileCk] + + if !dEx { + extraFiles += 1 + // file match + if b.ExtraFiles == nil { + b.ExtraFiles = make([]*DiskFile, 0) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + b.ExtraFiles = append(b.ExtraFiles, info) + } + + } + + // return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal + return sameFiles / (sameFiles + extraFiles + missingFiles) + +} + +func (d *Disk) GetPartialFileMatchesWithThreshold(t float64, filter []string) []*Disk { + + l := loggy.Get(0) + + var matchlist []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp") + if !exists { + return matchlist + } + + var lastPc int = -1 + for i, m := range matches { + //item := &Disk{} + if item, err := cache.Get(m); err == nil { + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc)) + } + + if item.FullPath != d.FullPath { + // only here if not looking at same disk + l.Logf(":: Checking overlapping files %s", item.Filename) + dSame := d.CompareFiles(item) + + item.MatchFactor = dSame + + if dSame >= t { + matchlist = append(matchlist, item) + } + } + + fmt.Print("\r") + lastPc = pc + } + } + + return matchlist +} + +func (d *Disk) HasFileSHA256(sha string) (bool, *DiskFile) { + + for _, file := range d.Files { + if sha == file.SHA256 { + return true, file + } + } + + return false, nil + +} + +func (d *Disk) GetFileChecksum(filename string) (bool, string) { + + for _, f := range d.Files { + if strings.ToLower(filename) == strings.ToLower(f.Filename) { + return true, f.SHA256 + } + } + + return false, "" + +} + +func (d *Disk) GetFileMatches(filename string, filter []string) []*Disk { + + l := loggy.Get(0) + + var matchlist []*Disk = make([]*Disk, 0) + + exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp") + if !exists { + return matchlist + } + + fileexists, SHA256 := d.GetFileChecksum(filename) + if !fileexists { + os.Stderr.WriteString("File does not exist on this volume: " + filename + "\n") + return matchlist + } + + _, srcFile := d.HasFileSHA256(SHA256) + + var lastPc int = -1 + for i, m := range matches { + //item := &Disk{} + if item, err := cache.Get(m); err == nil { + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc)) + } + + if item.FullPath != d.FullPath { + // only here if not looking at same disk + l.Logf(":: Checking overlapping files %s", item.Filename) + + if ex, file := item.HasFileSHA256(SHA256); ex { + if item.MatchFiles == nil { + item.MatchFiles = make(map[*DiskFile]*DiskFile) + } + item.MatchFiles[srcFile] = file + matchlist = append(matchlist, item) + } + } + + fmt.Print("\r") + lastPc = pc + } + } + + return matchlist +} + +// Gets directory with custom format +func (d *Disk) GetDirectory(format string) string { + out := "" + + for _, file := range d.Files { + + tmp := format + // size + tmp = strings.Replace(tmp, "{size:blocks}", fmt.Sprintf("%3d Blocks", file.Size/256+1), -1) + tmp = strings.Replace(tmp, "{size:kb}", fmt.Sprintf("%4d Kb", file.Size/1024+1), -1) + tmp = strings.Replace(tmp, "{size:b}", fmt.Sprintf("%6d Bytes", file.Size), -1) + tmp = strings.Replace(tmp, "{size}", fmt.Sprintf("%6d", file.Size), -1) + // format + tmp = strings.Replace(tmp, "{filename}", fmt.Sprintf("%-20s", file.Filename), -1) + // type + tmp = strings.Replace(tmp, "{type}", fmt.Sprintf("%-20s", file.Type), -1) + // sha256 + tmp = strings.Replace(tmp, "{sha256}", file.SHA256, -1) + // loadaddress + tmp = strings.Replace(tmp, "{loadaddr}", fmt.Sprintf("0x.%4X", file.LoadAddress), -1) + + out += tmp + "\n" + } + + return out +} + +type CacheContext int + +const ( + CC_All CacheContext = iota + CC_ActiveSectors + CC_AllSectors + CC_Files +) + +type DiskMetaDataCache struct { + ctx CacheContext + Disks map[string]*Disk +} + +var cache = NewCache(CC_All, "") + +func (c *DiskMetaDataCache) Get(filename string) (*Disk, error) { + cached, ok := c.Disks[filename] + if ok { + return cached, nil + } + item := &Disk{} + if err := item.ReadFromFile(filename); err == nil { + c.Disks[filename] = item + return item, nil + } + return nil, errors.New("Not found") +} + +func (c *DiskMetaDataCache) Put(filename string, item *Disk) { + c.Disks[filename] = item +} + +func NewCache(ctx CacheContext, pattern string) *DiskMetaDataCache { + + cache := &DiskMetaDataCache{ + ctx: ctx, + Disks: make(map[string]*Disk), + } + + return cache +} + +func CreateCache(ctx CacheContext, pattern string, filter []string) *DiskMetaDataCache { + + cache := &DiskMetaDataCache{ + ctx: ctx, + Disks: make(map[string]*Disk), + } + + exists, matches := existsPattern(*baseName, filter, pattern) + if !exists { + return cache + } + + var lastPc int = -1 + for i, m := range matches { + item := &Disk{} + if err := item.ReadFromFile(m); err == nil { + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("Caching data... %d%% ", pc)) + } + + // Load cache + cache.Put(m, item) + + fmt.Print("\r") + lastPc = pc + } + } + + return cache +} + +func SearchPartialFileMatchesWithThreshold(t float64, filter []string) map[string][2]*Disk { + + l := loggy.Get(0) + + matchlist := make(map[string][2]*Disk) + + exists, matches := existsPattern(*baseName, filter, "*_*_*_*.fgp") + if !exists { + return matchlist + } + + done := make(map[string]bool) + + var lastPc int = -1 + for i, m := range matches { + //item := &Disk{} + if disk, err := cache.Get(m); err == nil { + + d := *disk + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + os.Stderr.WriteString(fmt.Sprintf("Analyzing volumes... %d%% ", pc)) + } + + for _, n := range matches { + + if jj, err := cache.Get(n); err == nil { + + item := *jj + + key := d.SHA256 + ":" + item.SHA256 + if item.SHA256 < d.SHA256 { + key = item.SHA256 + ":" + d.SHA256 + } + + if _, ok := done[key]; ok { + continue + } + + if item.FullPath != d.FullPath { + // only here if not looking at same disk + l.Logf(":: Checking overlapping files %s", item.Filename) + dSame := d.CompareFiles(&item) + + item.MatchFactor = dSame + + if dSame >= t { + matchlist[key] = [2]*Disk{&d, &item} + } + } + + done[key] = true + + } + + } + + fmt.Print("\r") + lastPc = pc + } + } + + return matchlist +} + +const ingestWorkers = 4 +const processWorkers = 6 + +func exists(path string) bool { + + _, err := os.Stat(path) + if err != nil { + return false + } + return true + +} diff --git a/disk/atokens.go b/disk/atokens.go new file mode 100644 index 0000000..9bd5298 --- /dev/null +++ b/disk/atokens.go @@ -0,0 +1,757 @@ +package disk + +import ( + "fmt" + "strconv" + "strings" + + "regexp" +) + +//import "strings" + +var ApplesoftTokens = map[int]string{ + 0x80: "END", + 0x81: "FOR", + 0x82: "NEXT", + 0x83: "DATA", + 0x84: "INPUT", + 0x85: "DEL", + 0x86: "DIM", + 0x87: "READ", + 0x88: "GR", + 0x89: "TEXT", + 0x8A: "PR#", + 0x8B: "IN#", + 0x8C: "CALL", + 0x8D: "PLOT", + 0x8E: "HLIN", + 0x8F: "VLIN", + 0x90: "HGR2", + 0x91: "HGR", + 0x92: "HCOLOR=", + 0x93: "HPLOT", + 0x94: "DRAW", + 0x95: "XDRAW", + 0x96: "HTAB", + 0x97: "HOME", + 0x98: "ROT=", + 0x99: "SCALE=", + 0x9A: "SHLOAD", + 0x9B: "TRACE", + 0x9C: "NOTRACE", + 0x9D: "NORMAL", + 0x9E: "INVERSE", + 0x9F: "FLASH", + 0xA0: "COLOR=", + 0xA1: "POP", + 0xA2: "VTAB", + 0xA3: "HIMEM:", + 0xA4: "LOMEM:", + 0xA5: "ONERR", + 0xA6: "RESUME", + 0xA7: "RECALL", + 0xA8: "STORE", + 0xA9: "SPEED=", + 0xAA: "LET", + 0xAB: "GOTO", + 0xAC: "RUN", + 0xAD: "IF", + 0xAE: "RESTORE", + 0xAF: "&", + 0xB0: "GOSUB", + 0xB1: "RETURN", + 0xB2: "REM", + 0xB3: "STOP", + 0xB4: "ON", + 0xB5: "WAIT", + 0xB6: "LOAD", + 0xB7: "SAVE", + 0xB8: "DEF", + 0xB9: "POKE", + 0xBA: "PRINT", + 0xBB: "CONT", + 0xBC: "LIST", + 0xBD: "CLEAR", + 0xBE: "GET", + 0xBF: "NEW", + 0xC0: "TAB(", + 0xC1: "TO", + 0xC2: "FN", + 0xC3: "SPC(", + 0xC4: "THEN", + 0xC5: "AT", + 0xC6: "NOT", + 0xC7: "STEP", + 0xC8: "+", + 0xC9: "-", + 0xCA: "*", + 0xCB: "/", + 0xCC: "^", + 0xCD: "AND", + 0xCE: "OR", + 0xCF: ">", + 0xD0: "=", + 0xD1: "<", + 0xD2: "SGN", + 0xD3: "INT", + 0xD4: "ABS", + 0xD5: "USR", + 0xD6: "FRE", + 0xD7: "SCRN(", + 0xD8: "PDL", + 0xD9: "POS", + 0xDA: "SQR", + 0xDB: "RND", + 0xDC: "LOG", + 0xDD: "EXP", + 0xDE: "COS", + 0xDF: "SIN", + 0xE0: "TAN", + 0xE1: "ATN", + 0xE2: "PEEK", + 0xE3: "LEN", + 0xE4: "STR$", + 0xE5: "VAL", + 0xE6: "ASC", + 0xE7: "CHR$", + 0xE8: "LEFT$", + 0xE9: "RIGHT$", + 0xEA: "MID$", +} + +var ApplesoftReverse map[string]int +var IntegerReverse map[string]int + +func init() { + ApplesoftReverse = make(map[string]int) + IntegerReverse = make(map[string]int) + for k, v := range ApplesoftTokens { + ApplesoftReverse[v] = k + } + for k, v := range IntegerTokens { + IntegerReverse[v] = k + } + + tst() + +} + +var IntegerTokens = map[int]string{ + 0x00: "HIMEM:", + 0x02: "_", + 0x03: ":", + 0x04: "LOAD", + 0x05: "SAVE", + 0x06: "CON", + 0x07: "RUN", + 0x08: "RUN", + 0x09: "DEL", + 0x0A: ",", + 0x0B: "NEW", + 0x0C: "CLR", + 0x0D: "AUTO", + 0x0E: ",", + 0x0F: "MAN", + 0x10: "HIMEM:", + 0x11: "LOMEM:", + 0x12: "+", + 0x13: "-", + 0x14: "*", + 0x15: "/", + 0x16: "=", + 0x17: "#", + 0x18: ">=", + 0x19: ">", + 0x1A: "<=", + 0x1B: "<>", + 0x1C: "<", + 0x1D: "AND", + 0x1E: "OR", + 0x1F: "MOD", + 0x20: "^", + 0x21: "+", + 0x22: "(", + 0x23: ",", + 0x24: "THEN", + 0x25: "THEN", + 0x26: ",", + 0x27: ",", + 0x28: "\"", + 0x29: "\"", + 0x2A: "(", + 0x2B: "!", + 0x2C: "!", + 0x2D: "(", + 0x2E: "PEEK", + 0x2F: "RND", + 0x30: "SGN", + 0x31: "ABS", + 0x32: "PDL", + 0x33: "RNDX", + 0x34: "(", + 0x35: "+", + 0x36: "-", + 0x37: "NOT", + 0x38: "(", + 0x39: "=", + 0x3A: "#", + 0x3B: "LEN(", + 0x3C: "ASC(", + 0x3D: "SCRN(", + 0x3E: ",", + 0x3F: "(", + 0x40: "$", + 0x41: "$", + 0x42: "(", + 0x43: ",", + 0x44: ",", + 0x45: ";", + 0x46: ";", + 0x47: ";", + 0x48: ",", + 0x49: ",", + 0x4A: ",", + 0x4B: "TEXT", + 0x4C: "GR", + 0x4D: "CALL", + 0x4E: "DIM", + 0x4F: "DIM", + 0x50: "TAB", + 0x51: "END", + 0x52: "INPUT", + 0x53: "INPUT", + 0x54: "INPUT", + 0x55: "FOR", + 0x56: "=", + 0x57: "TO", + 0x58: "STEP", + 0x59: "NEXT", + 0x5A: ",", + 0x5B: "RETURN", + 0x5C: "GOSUB", + 0x5D: "REM", + 0x5E: "LET", + 0x5F: "GOTO", + 0x60: "IF", + 0x61: "PRINT", + 0x62: "PRINT", + 0x63: "PRINT", + 0x64: "POKE", + 0x65: ",", + 0x66: "COLOR=", + 0x67: "PLOT", + 0x68: ",", + 0x69: "HLIN", + 0x6A: ",", + 0x6B: "AT", + 0x6C: "VLIN", + 0x6D: ",", + 0x6E: "AT", + 0x6F: "VTAB", + 0x70: "=", + 0x71: "=", + 0x72: ")", + 0x73: ")", + 0x74: "LIST", + 0x75: ",", + 0x76: "LIST", + 0x77: "POP", + 0x78: "NODSP", + 0x79: "DSP", + 0x7A: "NOTRACE", + 0x7B: "DSP", + 0x7C: "DSP", + 0x7D: "TRACE", + 0x7E: "PR#", + 0x7F: "IN#", +} + +func Read16(srcptr, length *int, buffer []byte) int { + + // if *length < 2 { + // *srcptr += *length + // *length = 0 + // return 0 + // } + //fmt.Printf("-- srcptr=%d, length=%d, len(buffer)=%d\n", *srcptr, *length, len(buffer)) + + v := int(buffer[*srcptr]) + 256*int(buffer[*srcptr+1]) + + *srcptr += 2 + *length -= 2 + + return v + +} + +func Read8(srcptr, length *int, buffer []byte) byte { + + // if *length < 1 { + // *srcptr += *length + // *length = 0 + // return 0 + // } + //fmt.Printf("-- srcptr=%d, length=%d, len(buffer)=%d\n", *srcptr, *length, len(buffer)) + + v := buffer[*srcptr] + + *srcptr += 1 + *length -= 1 + + return v + +} + +func StripText(b []byte) []byte { + c := make([]byte, len(b)) + for i, v := range b { + c[i] = v & 127 + } + return c +} + +func ApplesoftDetoks(data []byte) []byte { + + //var baseaddr int = 0x801 + var srcptr int = 0x00 + var length int = len(data) + var out []byte = make([]byte, 0) + + if length < 2 { + // not enough here + return []byte("\r\n") + } + + for length > 0 { + + var nextAddr int + var lineNum int + var inQuote bool = false + var inRem bool = false + + if length < 2 { + break + } + + nextAddr = Read16(&srcptr, &length, data) + + if nextAddr == 0 { + break + } + + /* output line number */ + + if length < 2 { + break + } + + lineNum = Read16(&srcptr, &length, data) + ln := fmt.Sprintf("%d", lineNum) + + out = append(out, []byte(" "+ln+" ")...) + + if length == 0 { + break + } + + var t byte = Read8(&srcptr, &length, data) + + for t != 0 && length > 0 { + // process token + if t&0x80 != 0 { + /* token */ + tokstr, ok := ApplesoftTokens[int(t)] + if ok { + out = append(out, []byte(" "+tokstr+" ")...) + } else { + out = append(out, []byte(" ERROR ")...) + } + if t == 0xb2 { + inRem = true + } + } else { + /* simple character */ + r := rune(t) + if r == '"' && !inRem { + if !inQuote { + out = append(out, t) + } else { + out = append(out, t) + } + inQuote = !inQuote + } else if r == ':' && !inRem && !inQuote { + out = append(out, t) + } else if inRem && (r == '\r' || r == '\n') { + out = append(out, []byte("*")...) + } else { + out = append(out, t) + } + } + + // Advance + t = Read8(&srcptr, &length, data) + } + + out = append(out, []byte("\r\n")...) + + inQuote, inRem = false, false + + if length == 0 { + break + } + + } + + //fmt.Println(string(out)) + + return out + +} + +func IntegerDetoks(data []byte) []byte { + + var srcptr int = 0x00 + var length int = len(data) + var out []byte = make([]byte, 0) + + if length < 2 { + // not enough here + return []byte("\r\n") + } + + for length > 0 { + + // starting state for line + var lineLen byte + var lineNum int + var trailingSpace bool + var newTrailingSpace bool = false + + // read the line length + lineLen = Read8(&srcptr, &length, data) + + if lineLen == 0 { + break // zero length line found + } + + // read line number + lineNum = Read16(&srcptr, &length, data) + out = append(out, []byte(fmt.Sprintf("%d ", lineNum))...) + + // now process line + var t byte + t = Read8(&srcptr, &length, data) + for t != 0x01 && length > 0 { + if t == 0x03 { + out = append(out, []byte(" :")...) + t = Read8(&srcptr, &length, data) + } else if t == 0x28 { + /* start of quoted text */ + out = append(out, 34) + + t = Read8(&srcptr, &length, data) + for t != 0x29 && length > 0 { + out = append(out, t&0x7f) + t = Read8(&srcptr, &length, data) + } + if t != 0x29 { + break + } + + out = append(out, 34) + + t = Read8(&srcptr, &length, data) + } else if t == 0x5d { + /* start of REM statement, run to EOL */ + if trailingSpace { + out = append(out, 32) + } + out = append(out, []byte("REM ")...) + + t = Read8(&srcptr, &length, data) + for t != 0x01 && length > 0 { + out = append(out, t&0x7f) + t = Read8(&srcptr, &length, data) + } + if t != 0x01 { + break + } + } else if t >= 0xb0 && t <= 0xb9 { + /* start of integer constant */ + if length < 2 { + break + } + val := Read16(&srcptr, &length, data) + out = append(out, []byte(fmt.Sprintf("%d", val))...) + t = Read8(&srcptr, &length, data) + } else if t >= 0xc1 && t <= 0xda { + /* start of variable name */ + for (t >= 0xc1 && t <= 0xda) || (t >= 0xb0 && t <= 0xb9) { + /* note no RTF-escaped chars in this range */ + out = append(out, t&0x7f) + t = Read8(&srcptr, &length, data) + } + } else if t < 0x80 { + /* found a token; try to get the whitespace right */ + /* (maybe should've left whitespace on the ends of tokens + that are always followed by whitespace...?) */ + token, _ := IntegerTokens[int(t)] + if token[0] >= 0x21 && token[0] <= 0x3f || t < 0x12 { + /* does not need leading space */ + out = append(out, []byte(token)...) + } else { + /* needs leading space; combine with prev if it exists */ + if trailingSpace { + out = append(out, []byte(token)...) + } else { + out = append(out, []byte(" "+token)...) + } + out = append(out, 32) + } + if token[len(token)-1] == 32 { + newTrailingSpace = true + } + t = Read8(&srcptr, &length, data) + } else { + /* should not happen */ + t = Read8(&srcptr, &length, data) + } + + trailingSpace = newTrailingSpace + newTrailingSpace = false + } + + if t != 0x01 && length > 0 { + break // must have failed + } + + // ok, new line + out = append(out, []byte("\r\n")...) + + } + + return out + +} + +func breakingChar(ch rune) bool { + return ch == '(' || ch == ')' || ch == '.' || ch == ',' || ch == ';' || ch == ':' || ch == ' ' +} + +func ApplesoftTokenize(lines []string) []byte { + + start := 0x801 + currAddr := start + + buffer := make([]byte, 0) + + for _, l := range lines { + + l = strings.Trim(l, "\r") + if l == "" { + continue + } + + chunk := "" + inqq := false + tmp := strings.SplitN(l, " ", 2) + ln, _ := strconv.Atoi(tmp[0]) + rest := strings.Trim(tmp[1], " ") + + linebuffer := make([]byte, 4) + + // LINE NUMBER + linebuffer[0x02] = byte(ln & 0xff) + linebuffer[0x03] = byte(ln / 0x100) + + // PROCESS LINE + for _, ch := range rest { + + switch { + case inqq && ch != '"': + linebuffer = append(linebuffer, byte(ch)) + continue + case ch == '"': + linebuffer = append(linebuffer, byte(ch)) + inqq = !inqq + continue + case !inqq && breakingChar(ch): + linebuffer = append(linebuffer, []byte(chunk)...) + chunk = "" + linebuffer = append(linebuffer, byte(ch)) + continue + } + + chunk += string(ch) + code, ok := ApplesoftReverse[strings.ToUpper(chunk)] + if ok { + linebuffer = append(linebuffer, byte(code)) + chunk = "" + } + } + if chunk != "" { + linebuffer = append(linebuffer, []byte(chunk)...) + } + + // ENDING ZERO BYTE + linebuffer = append(linebuffer, 0x00) + + nextAddr := currAddr + len(linebuffer) + linebuffer[0x00] = byte(nextAddr & 0xff) + linebuffer[0x01] = byte(nextAddr / 0x100) + currAddr = nextAddr + + buffer = append(buffer, linebuffer...) + } + + buffer = append(buffer, 0x00, 0x00) + + return buffer + +} + +var reInt = regexp.MustCompile("^(-?[0-9]+)$") + +func isInt(s string) (bool, [3]byte) { + if reInt.MatchString(s) { + + m := reInt.FindAllStringSubmatch(s, -1) + i, _ := strconv.ParseInt(m[0][1], 10, 32) + return true, [3]byte{0xb9, byte(i % 256), byte(i / 256)} + + } else { + return false, [3]byte{0x00, 0x00, 0x00} + } +} + +func IntegerTokenize(lines []string) []byte { + + start := 0x801 + currAddr := start + + buffer := make([]byte, 0) + + var linebuffer []byte + + add := func(chunk string) { + if chunk != "" { + if ok, ival := isInt(chunk); ok { + linebuffer = append(linebuffer, ival[:]...) + //fmt.Printf("TOK Integer(%d)\n", int(ival[1])+256*int(ival[2])) + } else { + // Encode strings with high bit (0x80) set + //fmt.Printf("TOK String(%s)\n", strings.ToUpper(chunk)) + data := []byte(strings.ToUpper(chunk)) + for i, v := range data { + data[i] = v | 0x80 + } + linebuffer = append(linebuffer, data...) + } + } + } + + for _, l := range lines { + + l = strings.Trim(l, "\r") + if l == "" { + continue + } + + chunk := "" + inqq := false + tmp := strings.SplitN(l, " ", 2) + ln, _ := strconv.Atoi(tmp[0]) + rest := strings.Trim(tmp[1], " ") + + linebuffer = make([]byte, 3) + + // LINE NUMBER + linebuffer[0x01] = byte(ln & 0xff) + linebuffer[0x02] = byte(ln / 0x100) + + // PROCESS LINE + for _, ch := range rest { + + switch { + case inqq && ch != '"': + linebuffer = append(linebuffer, byte(ch|0x80)) + continue + case ch == ':' && !inqq: + linebuffer = append(linebuffer, 0x03) + continue + case ch == ',' && !inqq: + linebuffer = append(linebuffer, 0x0A) + continue + case ch == ';' && !inqq: + linebuffer = append(linebuffer, 0x45) + continue + case ch == '(' && !inqq: + linebuffer = append(linebuffer, 0x22) + continue + case ch == ')' && !inqq: + linebuffer = append(linebuffer, 0x72) + continue + case ch == '+' && !inqq: + linebuffer = append(linebuffer, 0x12) + continue + case ch == '"': + inqq = !inqq + if inqq { + ch = 0x28 + } else { + ch = 0x29 + } + linebuffer = append(linebuffer, byte(ch)) + continue + case !inqq && breakingChar(ch): + add(chunk) + chunk = "" + + //linebuffer = append(linebuffer, byte(ch|0x80)) + continue + } + + chunk += string(ch) + code, ok := IntegerReverse[strings.ToUpper(chunk)] + if ok { + //fmt.Printf("TOK Token(%s)\n", chunk) + linebuffer = append(linebuffer, byte(code)) + chunk = "" + } + } + if chunk != "" { + add(chunk) + } + + linebuffer = append(linebuffer, 0x01) // EOL token + + nextAddr := currAddr + len(linebuffer) + linebuffer[0x00] = byte(len(linebuffer)) + currAddr = nextAddr + + buffer = append(buffer, linebuffer...) + } + + // Encode file length + // buffer[0] = byte((len(buffer) - 2) % 256) + // buffer[1] = byte((len(buffer) - 2) / 256) + + return buffer + +} + +func tst() { + + // lines := []string{ + // "10 PRINT \"HELLO WORLD!\"", + // "20 GOTO 10", + // } + + // b := IntegerTokenize(lines) + + // Dump(b) + + // os.Exit(1) + +} diff --git a/disk/di_test.go b/disk/di_test.go new file mode 100644 index 0000000..01a1e42 --- /dev/null +++ b/disk/di_test.go @@ -0,0 +1,31 @@ +package disk + +import ( + //"strings" + "testing" +) +import "fmt" + +//import "os" + +func TestDisk(t *testing.T) { + + if STD_DISK_BYTES != 143360 { + t.Error(fmt.Sprintf("Wrong size got %d", STD_DISK_BYTES)) + } + + dsk, e := NewDSKWrapper("g19.dsk") + if e != nil { + t.Error(e) + } + + fmt.Printf("Disk format is %d\n", dsk.Format) + + _, fdlist, e := dsk.GetCatalogProDOSPathed(2, "GAMES", "") + for _, fd := range fdlist { + fmt.Printf("[%s]\n", fd.Name()) + } + + t.Fail() + +} diff --git a/disk/diskimage.go b/disk/diskimage.go new file mode 100644 index 0000000..e83a08c --- /dev/null +++ b/disk/diskimage.go @@ -0,0 +1,1090 @@ +package disk + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "strings" + + "crypto/sha256" + "encoding/hex" + + "fmt" +) + +//import "math/rand" + +const STD_BYTES_PER_SECTOR = 256 +const STD_TRACKS_PER_DISK = 35 +const STD_SECTORS_PER_TRACK = 16 +const STD_SECTORS_PER_TRACK_OLD = 13 +const STD_DISK_BYTES = STD_TRACKS_PER_DISK * STD_SECTORS_PER_TRACK * STD_BYTES_PER_SECTOR +const STD_DISK_BYTES_OLD = STD_TRACKS_PER_DISK * STD_SECTORS_PER_TRACK_OLD * STD_BYTES_PER_SECTOR +const PRODOS_800KB_BLOCKS = 1600 +const PRODOS_800KB_DISK_BYTES = STD_BYTES_PER_SECTOR * 2 * PRODOS_800KB_BLOCKS +const PRODOS_400KB_BLOCKS = 800 +const PRODOS_400KB_DISK_BYTES = STD_BYTES_PER_SECTOR * 2 * PRODOS_400KB_BLOCKS +const PRODOS_SECTORS_PER_BLOCK = 2 +const PRODOS_BLOCKS_PER_TRACK = 8 +const PRODOS_800KB_BLOCKS_PER_TRACK = 20 +const PRODOS_BLOCKS_PER_DISK = 280 +const PRODOS_ENTRY_SIZE = 39 + +const TRACK_NIBBLE_LENGTH = 0x1A00 +const TRACK_COUNT = STD_TRACKS_PER_DISK +const SECTOR_COUNT = STD_SECTORS_PER_TRACK +const HALF_TRACK_COUNT = TRACK_COUNT * 2 +const DISK_NIBBLE_LENGTH = TRACK_NIBBLE_LENGTH * TRACK_COUNT +const DISK_PLAIN_LENGTH = STD_DISK_BYTES +const DISK_2MG_NON_NIB_LENGTH = DISK_PLAIN_LENGTH + 0x040 +const DISK_2MG_NIB_LENGTH = DISK_NIBBLE_LENGTH + 0x040 + +type DSKContainer []byte + +func Checksum(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +type SectorOrder int + +var DOS_33_SECTOR_ORDER = []int{ + 0x00, 0x07, 0x0E, 0x06, 0x0D, 0x05, 0x0C, 0x04, + 0x0B, 0x03, 0x0A, 0x02, 0x09, 0x01, 0x08, 0x0F, +} + +var DOS_32_SECTOR_ORDER = []int{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, +} + +var PRODOS_SECTOR_ORDER = []int{ + 0x00, 0x08, 0x01, 0x09, 0x02, 0x0a, 0x03, 0x0b, + 0x04, 0x0c, 0x05, 0x0d, 0x06, 0x0e, 0x07, 0x0f, +} +var LINEAR_SECTOR_ORDER = []int{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +} + +var NIBBLE_62 = []byte{ + 0x96, 0x97, 0x9a, 0x9b, 0x9d, 0x9e, 0x9f, 0xa6, + 0xa7, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb2, 0xb3, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb9, 0xba, 0xbb, 0xbc, + 0xbd, 0xbe, 0xbf, 0xcb, 0xcd, 0xce, 0xcf, 0xd3, + 0xd6, 0xd7, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, + 0xdf, 0xe5, 0xe6, 0xe7, 0xe9, 0xea, 0xeb, 0xec, + 0xed, 0xee, 0xef, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, + 0xf7, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff} + +var NIBBLE_53 = []byte{ + 0xab, 0xad, 0xae, 0xaf, 0xb5, 0xb6, 0xb7, 0xba, + 0xbb, 0xbd, 0xbe, 0xbf, 0xd6, 0xd7, 0xda, 0xdb, + 0xdd, 0xde, 0xdf, 0xea, 0xeb, 0xed, 0xee, 0xef, + 0xf5, 0xf6, 0xf7, 0xfa, 0xfb, 0xfd, 0xfe, 0xff, +} + +var identity = map[string]DiskFormat{ + "99b900080a0a0a990008c8d0f4a62ba9098527adcc03854184408a4a4a4a4a09": GetDiskFormat(DF_DOS_SECTORS_13), + "01a527c909d018a52b4a4a4a4a09c0853fa95c853e18adfe086dff088dfe08ae": GetDiskFormat(DF_DOS_SECTORS_16), + "0138b0034c32a18643c903088a29704a4a4a4a09c08549a0ff844828c8b148d0": GetDiskFormat(DF_PRODOS), +} + +const ( + SectorOrderDOS33 SectorOrder = iota + SectorOrderDOS32 + SectorOrderDOS33Alt + SectorOrderProDOS + SectorOrderProDOSLinear +) + +func (so SectorOrder) String() string { + switch so { + case SectorOrderDOS32: + return "DOS" + case SectorOrderDOS33: + return "DOS" + case SectorOrderDOS33Alt: + return "DOS Alternate" + case SectorOrderProDOS: + return "ProDOS" + case SectorOrderProDOSLinear: + return "Linear" + } + + return "Linear" +} + +type DiskFormatID int + +const ( + DF_NONE DiskFormatID = iota + DF_DOS_SECTORS_13 + DF_DOS_SECTORS_16 + DF_PRODOS + DF_PRODOS_800KB + DF_PASCAL + DF_RDOS_3 + DF_RDOS_32 + DF_RDOS_33 + DF_PRODOS_400KB + DF_PRODOS_CUSTOM +) + +type DiskFormat struct { + ID DiskFormatID + bpd int + spt int + uspt int + tpd int +} + +func GetDiskFormat(id DiskFormatID) DiskFormat { + return DiskFormat{ID: id} +} + +func GetPDDiskFormat(id DiskFormatID, blocks int) DiskFormat { + return DiskFormat{ + ID: id, + bpd: blocks, + tpd: 80, + spt: blocks / 80, + uspt: blocks / 80, + } +} + +func (f DiskFormat) String() string { + switch f.ID { + case DF_NONE: + return "Unrecognized" + case DF_DOS_SECTORS_13: + return "Apple DOS 13 Sector" + case DF_DOS_SECTORS_16: + return "Apple DOS 16 Sector" + case DF_PRODOS: + return "ProDOS" + case DF_PASCAL: + return "Pascal" + case DF_PRODOS_400KB: + return "ProDOS 400Kb" + case DF_PRODOS_800KB: + return "ProDOS 800Kb" + case DF_RDOS_3: + return "SSI RDOS 3 (16/13/Physical)" + case DF_RDOS_32: + return "SSI RDOS 32 (13/13/Physical)" + case DF_RDOS_33: + return "SSI RDOS 32 (16/16/PD)" + case DF_PRODOS_CUSTOM: + return fmt.Sprintf("ProDOS Custom (%d SPT, %d TPD)", f.SPT(), f.TPD()) + } + return "Unrecognized" +} + +func (df DiskFormat) BPD() int { + switch df.ID { + case DF_RDOS_3: + return 222 + case DF_RDOS_32: + return 222 + case DF_RDOS_33: + return 280 + case DF_DOS_SECTORS_13: + return 222 + case DF_DOS_SECTORS_16: + return 280 + case DF_PRODOS: + return 280 + case DF_PASCAL: + return 280 + case DF_PRODOS_800KB: + return 1600 + case DF_PRODOS_400KB: + return 800 + case DF_PRODOS_CUSTOM: + return df.bpd + } + return 16 // fallback +} + +func (df DiskFormat) USPT() int { + switch df.ID { + case DF_RDOS_3: + return 13 + case DF_RDOS_32: + return 13 + case DF_RDOS_33: + return 16 + case DF_DOS_SECTORS_13: + return 13 + case DF_DOS_SECTORS_16: + return 16 + case DF_PRODOS: + return 16 + case DF_PASCAL: + return 16 + case DF_PRODOS_800KB: + return 40 + case DF_PRODOS_400KB: + return 20 + case DF_PRODOS_CUSTOM: + return df.uspt + } + return 16 // fallback +} + +func (df DiskFormat) SPT() int { + switch df.ID { + case DF_RDOS_3: + return 16 + case DF_RDOS_32: + return 13 + case DF_RDOS_33: + return 16 + case DF_DOS_SECTORS_13: + return 13 + case DF_DOS_SECTORS_16: + return 16 + case DF_PRODOS: + return 16 + case DF_PASCAL: + return 16 + case DF_PRODOS_800KB: + return 40 + case DF_PRODOS_400KB: + return 20 + case DF_PRODOS_CUSTOM: + return df.spt + } + return 16 // fallback +} + +func (df DiskFormat) TPD() int { + switch df.ID { + case DF_RDOS_3: + return 35 + case DF_RDOS_32: + return 35 + case DF_RDOS_33: + return 35 + case DF_DOS_SECTORS_13: + return 35 + case DF_DOS_SECTORS_16: + return 35 + case DF_PRODOS: + return 35 + case DF_PASCAL: + return 35 + case DF_PRODOS_800KB: + return 80 + case DF_PRODOS_400KB: + return 80 + case DF_PRODOS_CUSTOM: + return df.tpd + } + return 35 // fallback +} + +type Nibbler interface { + SetNibble(offset int, value byte) + GetNibble(offset int) byte +} + +type DSKWrapper struct { + Data DSKContainer + Layout SectorOrder + CurrentTrack int + CurrentSector int + SectorPointer int + Format DiskFormat + RDOSFormat RDOSFormat + Filename string + //Nibbles []byte + Nibbles Nibbler + CurrentSectorOrder []int + WriteProtected bool + NibblesChanged bool +} + +// SectoreMapperDOS33 handles the interleaving for dos sectors +func SectorMapperDOS33(wanted int) int { + + return wanted + + switch wanted { + case 0: + return 0 + case 13: + return 1 + case 11: + return 2 + case 9: + return 3 + case 7: + return 4 + case 5: + return 5 + case 3: + return 6 + case 1: + return 7 + case 14: + return 8 + case 12: + return 9 + case 10: + return 10 + case 8: + return 11 + case 6: + return 12 + case 4: + return 13 + case 2: + return 14 + case 15: + return 15 + } + + return -1 // invalid sector +} + +func SectorMapperDOS33Alt(wanted int) int { + + switch wanted { + case 0: + return 0 + case 13: + return 1 + case 11: + return 2 + case 9: + return 3 + case 7: + return 4 + case 5: + return 5 + case 3: + return 6 + case 1: + return 7 + case 14: + return 8 + case 12: + return 9 + case 10: + return 10 + case 8: + return 11 + case 6: + return 12 + case 4: + return 13 + case 2: + return 14 + case 15: + return 15 + } + + return -1 // invalid sector +} + +func SectorMapperDOS33bad(wanted int) int { + + return wanted + + switch wanted { + case 0: + return 0 + case 1: + return 13 + case 2: + return 11 + case 3: + return 9 + case 4: + return 7 + case 5: + return 5 + case 6: + return 3 + case 7: + return 1 + case 8: + return 14 + case 9: + return 12 + case 10: + return 10 + case 11: + return 8 + case 12: + return 6 + case 13: + return 4 + case 14: + return 2 + case 15: + return 15 + } + + return -1 // invalid sector +} + +// SectoreMapperProDOS handles the interleaving for dos sectors +func SectorMapperProDOS(wanted int) int { + switch wanted { + case 0: + return 0 + case 1: + return 2 + case 2: + return 4 + case 3: + return 6 + case 4: + return 8 + case 5: + return 10 + case 6: + return 12 + case 7: + return 14 + case 8: + return 1 + case 9: + return 3 + case 10: + return 5 + case 11: + return 7 + case 12: + return 9 + case 13: + return 11 + case 14: + return 13 + case 15: + return 15 + } + + return -1 // invalid sector +} + +func (d *DSKWrapper) SetTrack(t int) error { + + if t >= 0 && t < d.Format.TPD() { + d.CurrentTrack = t + d.SetSectorPointer() + return nil + } + + return errors.New("Invalid track") + +} + +// SetSector changes the sector we are looking at +func (d *DSKWrapper) SetSector(s int) error { + if s >= 0 && s < d.Format.USPT() { + d.CurrentSector = s + d.SetSectorPointer() + return nil + } + + return errors.New("Invalid sector") +} + +func (d *DSKWrapper) HuntVTOC(t, s int) (int, int) { + for block := 0; block < len(d.Data)/256; block++ { + data := d.Data[block*256 : block*256+256] + var v VTOC + v.SetData(data, (block / s), (block % s)) + if v.GetTracks() == t && v.GetSectors() == s { + return (block / s), (block % s) + } + } + return -1, -1 +} + +// SetSectorPointer calculates the pointer to the current sector, taking into +// account the sector interleaving of the DSK image. +func (d *DSKWrapper) SetSectorPointer() { + + track := d.CurrentTrack + sector := d.CurrentSector + isector := sector + switch d.Layout { + case SectorOrderDOS33Alt: + isector = SectorMapperDOS33Alt(sector) + case SectorOrderDOS33: + isector = SectorMapperDOS33(sector) + case SectorOrderProDOS: + isector = SectorMapperProDOS(sector) + } + + d.SectorPointer = (track * d.Format.SPT() * STD_BYTES_PER_SECTOR) + (STD_BYTES_PER_SECTOR * isector) +} + +func (d *DSKWrapper) UpdateTrack(track int) { + d.NibblesChanged = true +} + +func (d *DSKWrapper) ChecksumDisk() string { + return Checksum(d.Data) +} + +func (d *DSKWrapper) ChecksumSector(t, s int) string { + d.SetTrack(t) + d.SetSector(s) + return Checksum(d.Data[d.SectorPointer : d.SectorPointer+256]) +} + +func (d *DSKWrapper) IsChanged() bool { + return d.NibblesChanged +} + +// func (d *DSKWrapper) GetNibbles() []byte { +// return d.Nibbles +// } + +// Read is a simple function to return the current pointed to sector +func (d *DSKWrapper) Read() []byte { + ////fmt.Printf("---> Reading track %d, sector %d\n", d.CurrentTrack, d.CurrentSector) + return d.Data[d.SectorPointer : d.SectorPointer+256] +} + +func (d *DSKWrapper) Write(data []byte) { + ////fmt.Printf("---> Reading track %d, sector %d\n", d.CurrentTrack, d.CurrentSector) + l := len(data) + if l > STD_BYTES_PER_SECTOR { + l = STD_BYTES_PER_SECTOR + } + + for i, v := range data { + if i >= l { + break + } + d.Data[d.SectorPointer+i] = v + } +} + +// Seek is a convienience function to go straight to a particular track & sector +func (d *DSKWrapper) Seek(t, s int) error { + + var e error + + e = d.SetTrack(t) + if e != nil { + return e + } + + e = d.SetSector(s) + + return e +} + +func (d *DSKWrapper) SetData(data []byte) { + // for i, v := range data { + // d.Data[i] = v + // } + d.Data = data +} + +func NewDSKWrapper(nibbler Nibbler, filename string) (*DSKWrapper, error) { + + f, e := os.Open(filename) + if e != nil { + return nil, e + } + data, e := ioutil.ReadAll(f) + f.Close() + if e != nil { + return nil, e + } + + w, e := NewDSKWrapperBin(nibbler, data, filename) + return w, e + +} + +func NewDSKWrapperBin(nibbler Nibbler, data []byte, filename string) (*DSKWrapper, error) { + + if len(data) != 232960 && + len(data) != STD_DISK_BYTES && + len(data) != STD_DISK_BYTES_OLD && + len(data) != PRODOS_400KB_DISK_BYTES && + len(data) != PRODOS_400KB_DISK_BYTES+64 && + len(data) != PRODOS_800KB_DISK_BYTES && + len(data) != PRODOS_800KB_DISK_BYTES+64 && + len(data) != STD_DISK_BYTES+64 { + return nil, errors.New("Incorrect disk bytes") + } + + this := &DSKWrapper{} + + this.SetData(data) + this.Filename = filename + this.Layout = SectorOrderDOS33 + this.CurrentSectorOrder = DOS_33_SECTOR_ORDER + this.Nibbles = nibbler + this.WriteProtected = false + + this.Identify() + + return this, nil + +} + +func (dsk *DSKWrapper) GetNibbles() []byte { + + n := make([]byte, DISK_NIBBLE_LENGTH) + for i, _ := range n { + n[i] = dsk.Nibbles.GetNibble(i) + } + + return n + +} + +func (dsk *DSKWrapper) SetNibbles(data []byte) { + if dsk.Nibbles == nil { + return + } + for i, v := range data { + dsk.Nibbles.SetNibble(i, v) + } +} + +func (dsk *DSKWrapper) Identify() { + + dsk.Format = GetDiskFormat(DF_NONE) + + var hint DiskFormat + + dsk.Filename = strings.ToLower(dsk.Filename) + + switch { + case strings.HasSuffix(dsk.Filename, ".po"): + hint = GetDiskFormat(DF_PRODOS) + case strings.HasSuffix(dsk.Filename, ".do"): + hint = GetDiskFormat(DF_DOS_SECTORS_16) + default: + hint = GetDiskFormat(DF_DOS_SECTORS_16) + } + + is2MG, Format, Layout, zdsk := dsk.Is2MG() + if is2MG { + ////fmt.Println("repacked", len(zdsk.Data)) + dsk.SetData(zdsk.Data) + dsk.Layout = Layout + dsk.Format = Format + return + } + + isPD, Format, Layout := dsk.IsProDOS() + if isPD { + if Format == GetDiskFormat(DF_PRODOS) { + dsk.Format = GetDiskFormat(DF_PRODOS) + dsk.Layout = Layout + switch dsk.Layout { + case SectorOrderProDOS: + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + case SectorOrderProDOSLinear: + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + case SectorOrderDOS33: + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + } + dsk.SetNibbles(dsk.Nibblize()) + return + } else { + dsk.Format = GetDiskFormat(DF_PRODOS_800KB) + dsk.Layout = Layout + switch dsk.Layout { + case SectorOrderProDOS: + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + case SectorOrderProDOSLinear: + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + case SectorOrderDOS33: + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + } + dsk.SetNibbles(make([]byte, 232960)) + return + } + } + + isRDOS, Version := dsk.IsRDOS() + if isRDOS { + dsk.RDOSFormat = Version + switch Version { + case RDOS_3: + dsk.Format = GetDiskFormat(DF_RDOS_3) + dsk.Layout = SectorOrderDOS33Alt + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + case RDOS_32: + dsk.Format = GetDiskFormat(DF_RDOS_32) + dsk.Layout = SectorOrderDOS33Alt + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.SetNibbles(make([]byte, 232960)) // FIXME: fix nibbles + case RDOS_33: + dsk.Format = GetDiskFormat(DF_RDOS_33) + dsk.Layout = SectorOrderProDOS + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + } + return + } + + isAppleDOS, Format, Layout := dsk.IsAppleDOS() + if isAppleDOS { + ////fmt.Printf("Format: %s\n", Format.String()) + dsk.Format = Format + dsk.Layout = Layout + switch Layout { + case SectorOrderProDOS: + ////fmt.Println("Sector Order: ProDOS") + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + case SectorOrderProDOSLinear: + ////fmt.Println("Sector Order: Linear") + dsk.CurrentSectorOrder = LINEAR_SECTOR_ORDER + case SectorOrderDOS33: + ////fmt.Println("Sector Order: DOS33") + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + case SectorOrderDOS32: + ////fmt.Println("Sector Order: DOS32") + //dsk.CurrentSectorOrder = DOS_32_SECTOR_ORDER + case SectorOrderDOS33Alt: + ////fmt.Println("Sector Order: Alt Linear") + dsk.CurrentSectorOrder = LINEAR_SECTOR_ORDER + } + dsk.SetNibbles(dsk.Nibblize()) + return + } + + fp := hex.EncodeToString(dsk.Data[:32]) + if dfmt, ok := identity[fp]; ok { + dsk.Format = dfmt + //fmt.Println(dsk.Format.String()) + } + + //fmt.Printf("Disk name: %s\n", dsk.Filename) + + // 1. NIB + if len(dsk.Data) == 232960 { + //fmt.Println("THIS IS A NIB") + dsk.Format = GetDiskFormat(DF_DOS_SECTORS_16) + dsk.SetNibbles(dsk.Data) + return + } + + // 2. Wrong size + if len(dsk.Data) != STD_DISK_BYTES && len(dsk.Data) != STD_DISK_BYTES_OLD && len(dsk.Data) != PRODOS_800KB_DISK_BYTES { + //fmt.Println("NOT STANDARD DISK") + dsk.Format = GetDiskFormat(DF_NONE) + dsk.SetNibbles(make([]byte, 232960)) + return + } + + // 3. DOS 3x Disk + vtoc, e := dsk.AppleDOSGetVTOC() + //fmt.Println(vtoc.GetTracks(), vtoc.GetSectors()) + if e == nil && vtoc.GetTracks() == 35 { + //bps := vtoc.BytesPerSector() + t := vtoc.GetTracks() + s := vtoc.GetSectors() + + //fmt.Printf("DOS Tracks = %d, Sectors = %d\n", t, s) + + if t == 35 && s == 16 { + dsk.Layout = SectorOrderDOS33 + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.Format = GetDiskFormat(DF_DOS_SECTORS_16) + dsk.SetNibbles(dsk.Nibblize()) + } else if t == 35 && s == 13 { + dsk.Layout = SectorOrderDOS32 + dsk.CurrentSectorOrder = DOS_32_SECTOR_ORDER + dsk.Format = GetDiskFormat(DF_DOS_SECTORS_13) + dsk.SetNibbles(make([]byte, 232960)) + } + return + } + + //fmt.Println("Trying prodos / pascal") + + isPAS, volName := dsk.IsPascal() + if isPAS && volName != "" { + dsk.Format = GetDiskFormat(DF_PASCAL) + dsk.Layout = SectorOrderDOS33 + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + return + } + + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + + if dsk.Format == GetDiskFormat(DF_PRODOS) && len(dsk.Data) == PRODOS_800KB_DISK_BYTES { + dsk.Format = GetDiskFormat(DF_PRODOS_800KB) + } + + //fmt.Println(dsk.Format.String()) + + switch dsk.Format.ID { + + case DF_PRODOS: + + vdh, e := dsk.PRODOSGetVDH(2) + ////fmt.Printf("Blocks = %d\n", vdh.GetTotalBlocks()) + if e == nil && vdh.GetStorageType() == 0xf && vdh.GetTotalBlocks() == 280 { + dsk.Format = GetDiskFormat(DF_PRODOS) + dsk.Layout = SectorOrderDOS33 + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + + //fmt.Println("THIS IS A PRODOS DISKETTE, DOS Ordered") + return + } else { + dsk.Format = GetDiskFormat(DF_PRODOS) + dsk.Layout = SectorOrderDOS33Alt + + ////fmt.Println("Try again") + + vdh, e = dsk.PRODOSGetVDH(2) + + if e == nil && vdh.GetStorageType() == 0xf && vdh.GetTotalBlocks() == 280 { + + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + + //fmt.Println("THIS IS A PRODOS DISKETTE, ProDOS Ordered") + return + + } + } + + case DF_PRODOS_800KB: + + vdh, e := dsk.PRODOS800GetVDH(2) + //fmt.Printf("Blocks = %d\n", vdh.GetTotalBlocks()) + if e == nil && vdh.GetStorageType() == 0xf && vdh.GetTotalBlocks() == 1600 { + dsk.Format = GetDiskFormat(DF_PRODOS_800KB) + dsk.Layout = SectorOrderDOS33 + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + //dsk.SetNibbles(dsk.nibblize()) + dsk.SetNibbles(make([]byte, 232960)) + + //fmt.Println("THIS IS A PRODOS DISKETTE, DOS Ordered") + return + } + + } + + switch hint.ID { + case DF_PRODOS: + dsk.Format = GetDiskFormat(DF_PRODOS) + dsk.Layout = SectorOrderProDOS + dsk.CurrentSectorOrder = PRODOS_SECTOR_ORDER + dsk.SetNibbles(dsk.Nibblize()) + //fmt.Println("VTOC read failed, will nibblize anyway...") + case DF_DOS_SECTORS_16: + //fmt.Println("VTOC read failed, will nibblize anyway...") + dsk.Layout = SectorOrderProDOS + dsk.CurrentSectorOrder = DOS_33_SECTOR_ORDER + dsk.Format = GetDiskFormat(DF_DOS_SECTORS_16) + dsk.SetNibbles(dsk.Nibblize()) + } + +} + +//func (d *DSKWrapper) ReadFileSectorsProDOS(fd ProDOSFileDescriptor) ([]byte, error) { + +//} + +func Dump(bytes []byte) { + perline := 0xC + base := 0 + ascii := "" + for i, v := range bytes { + if i%perline == 0 { + fmt.Println(" " + ascii) + ascii = "" + fmt.Printf("%.4X:", base+i) + } + if v >= 32 && v < 128 { + ascii += string(rune(v)) + } else { + ascii += "." + } + fmt.Printf(" %.2X", v) + } + fmt.Println(" " + ascii) +} + +func Between(v, lo, hi uint) bool { + return ((v >= lo) && (v <= hi)) +} + +func PokeToAscii(v uint, usealt bool) int { + highbit := v & 1024 + + v = v & 1023 + + if Between(v, 0, 31) { + return int((64 + (v % 32)) | highbit) + } + + if Between(v, 32, 63) { + return int((32 + (v % 32)) | highbit) + } + + if Between(v, 64, 95) { + if usealt { + return int((128 + (v % 32)) | highbit) + } else { + return int((64 + (v % 32)) | highbit) + } + } + + if Between(v, 96, 127) { + if usealt { + return int((96 + (v % 32)) | highbit) + } else { + return int((32 + (v % 32)) | highbit) + } + } + + if Between(v, 128, 159) { + return int((64 + (v % 32)) | highbit) + } + + if Between(v, 160, 191) { + return int((32 + (v % 32)) | highbit) + } + + if Between(v, 192, 223) { + return int((64 + (v % 32)) | highbit) + } + + if Between(v, 224, 255) { + return int((96 + (v % 32)) | highbit) + } + + return int(v | highbit) +} + +func (d *DSKWrapper) Nibblize() []byte { + + if len(d.Data) != STD_DISK_BYTES { + return make([]byte, 232960) + } + + data := d.Data + + output := bytes.NewBuffer([]byte(nil)) + + for track := 0; track < STD_TRACKS_PER_DISK; track++ { + //d.writeJunkBytes(output, 48); + for sector := 0; sector < STD_SECTORS_PER_TRACK; sector++ { + //gap2 := int((rand.Float32() * 5.0) + 4) + gap2 := 6 + // 15 junk bytes + d.writeJunkBytes(output, 15) + // Address block + d.writeAddressBlock(output, track, sector, 254) + // 4 junk bytes + d.writeJunkBytes(output, gap2) + // Data block + d.nibblizeBlock(output, track, d.CurrentSectorOrder[sector], data) + // 34 junk bytes + d.writeJunkBytes(output, 38-gap2) + } + } + + return output.Bytes() + +} + +func (d *DSKWrapper) NibbleOffsetToTS(offset int) (int, int) { + offset = offset - (offset % 256) + c := offset / 256 + sector := c % SECTOR_COUNT + track := (c - sector) / SECTOR_COUNT + return track, sector +} + +func (d *DSKWrapper) nibblizeBlock(output io.Writer, track, sector int, nibbles []byte) { + + //log.Printf("NibblizeBlock(%d, %d)", track, sector) + + offset := ((track * SECTOR_COUNT) + sector) * 256 + temp := make([]int, 342) + for i := 0; i < 256; i++ { + temp[i] = int((nibbles[offset+i] & 0x0ff) >> 2) + } + hi := 0x001 + med := 0x0AB + low := 0x055 + + for i := 0; i < 0x56; i++ { + value := ((nibbles[offset+hi] & 1) << 5) | + ((nibbles[offset+hi] & 2) << 3) | + ((nibbles[offset+med] & 1) << 3) | + ((nibbles[offset+med] & 2) << 1) | + ((nibbles[offset+low] & 1) << 1) | + ((nibbles[offset+low] & 2) >> 1) + temp[i+256] = int(value) + hi = (hi - 1) & 0x0ff + med = (med - 1) & 0x0ff + low = (low - 1) & 0x0ff + } + output.Write([]byte{0x0d5, 0x0aa, 0x0ad}) + + last := 0 + for i := len(temp) - 1; i > 255; i-- { + value := temp[i] ^ last + output.Write([]byte{NIBBLE_62[value]}) + last = temp[i] + } + for i := 0; i < 256; i++ { + value := temp[i] ^ last + output.Write([]byte{NIBBLE_62[value]}) + last = temp[i] + } + // Last data byte used as checksum + output.Write([]byte{NIBBLE_62[last]}) + output.Write([]byte{0x0de, 0x0aa, 0x0eb}) + +} + +func (d *DSKWrapper) writeJunkBytes(output io.Writer, i int) { + for c := 0; c < i; c++ { + output.Write([]byte{0xff}) + } +} + +func (d *DSKWrapper) writeAddressBlock(output io.Writer, track, sector int, volumeNumber int) { + output.Write([]byte{0x0d5, 0x0aa, 0x096}) + + var checksum int = 0x00 + // volume + checksum ^= volumeNumber + output.Write(d.getOddEven(volumeNumber)) + // track + checksum ^= track + output.Write(d.getOddEven(track)) + // sector + checksum ^= sector + output.Write(d.getOddEven(sector)) + // checksum + output.Write(d.getOddEven(checksum & 0x0ff)) + + output.Write([]byte{0xde, 0xaa, 0xeb}) +} + +func (d *DSKWrapper) getOddEven(i int) []byte { + out := []byte{0, 0} + out[0] = byte(0xAA | (i >> 1)) + out[1] = byte(0xAA | i) + return out +} diff --git a/disk/diskimage2mg.go b/disk/diskimage2mg.go new file mode 100644 index 0000000..c555581 --- /dev/null +++ b/disk/diskimage2mg.go @@ -0,0 +1,106 @@ +package disk + +import "fmt" + +/* + 2MG format loader... +*/ + +const PREAMBLE_2MG_SIZE = 0x40 + +var MAGIC_2MG = []byte{byte('2'), byte('I'), byte('M'), byte('G')} + +type Header2MG struct { + Data [64]byte +} + +func (h *Header2MG) SetData(data []byte) { + for i, v := range data { + if i < 64 { + h.Data[i] = v + } + } +} + +func (h *Header2MG) GetID() string { + return string(h.Data[0x00:0x04]) +} + +func (h *Header2MG) GetCreatorID() string { + return string(h.Data[0x04:0x08]) +} + +func (h *Header2MG) GetHeaderSize() int { + return int(h.Data[0x08]) + 256*int(h.Data[0x09]) +} + +func (h *Header2MG) GetVersion() int { + return int(h.Data[0x0A]) + 256*int(h.Data[0x0B]) +} + +func (h *Header2MG) GetImageFormat() int { + return int(h.Data[0x0C]) + 256*int(h.Data[0x0D]) + 65336*int(h.Data[0x0E]) + 16777216*int(h.Data[0x0F]) +} + +func (h *Header2MG) GetDOSFlags() int { + return int(h.Data[0x10]) + 256*int(h.Data[0x11]) + 65336*int(h.Data[0x12]) + 16777216*int(h.Data[0x13]) +} + +func (h *Header2MG) GetProDOSBlocks() int { + return int(h.Data[0x14]) + 256*int(h.Data[0x15]) + 65336*int(h.Data[0x16]) + 16777216*int(h.Data[0x17]) +} + +func (h *Header2MG) GetDiskDataStart() int { + return int(h.Data[0x18]) + 256*int(h.Data[0x19]) + 65336*int(h.Data[0x1A]) + 16777216*int(h.Data[0x1B]) +} + +func (h *Header2MG) GetDiskDataLength() int { + return int(h.Data[0x1C]) + 256*int(h.Data[0x1D]) + 65336*int(h.Data[0x1E]) + 16777216*int(h.Data[0x1F]) +} + +func (dsk *DSKWrapper) Is2MG() (bool, DiskFormat, SectorOrder, *DSKWrapper) { + + h := &Header2MG{} + h.SetData(dsk.Data[:0x40]) + + if h.GetID() != "2IMG" { + return false, GetDiskFormat(DF_NONE), SectorOrderDOS33, nil + } + + fmt.Println("Disk has 2MG Magic") + fmt.Printf("Block count %d\n", h.GetProDOSBlocks()) + + start := h.GetDiskDataStart() + size := h.GetDiskDataLength() + + if size < len(dsk.Data)-start { + size = len(dsk.Data) - start + } + + if size != STD_DISK_BYTES && size != PRODOS_800KB_DISK_BYTES && size != PRODOS_400KB_DISK_BYTES { + fmt.Printf("Bad size %d bytes @ start %d\n", size, start) + return false, GetDiskFormat(DF_NONE), SectorOrderDOS33, nil + } + + data := dsk.Data[start : start+size] + format := h.GetImageFormat() + switch format { + case 0x00: /* DOS sector order */ + zdsk, _ := NewDSKWrapperBin(dsk.Nibbles, data, dsk.Filename) + return true, GetDiskFormat(DF_DOS_SECTORS_16), SectorOrderDOS33, zdsk + case 0x01: /* ProDOS sector order */ + zdsk, _ := NewDSKWrapperBin(dsk.Nibbles, data, dsk.Filename) + + if h.GetProDOSBlocks() == 1600 { + return true, GetDiskFormat(DF_PRODOS_800KB), SectorOrderProDOSLinear, zdsk + } else if h.GetProDOSBlocks() == 800 { + return true, GetDiskFormat(DF_PRODOS_400KB), SectorOrderProDOSLinear, zdsk + } else { + return true, GetPDDiskFormat(DF_PRODOS_CUSTOM, h.GetProDOSBlocks()), SectorOrderProDOSLinear, zdsk + } + + } + + return false, GetDiskFormat(DF_NONE), SectorOrderDOS33, nil + +} diff --git a/disk/diskimageappledos.go b/disk/diskimageappledos.go new file mode 100644 index 0000000..aa66c03 --- /dev/null +++ b/disk/diskimageappledos.go @@ -0,0 +1,1136 @@ +package disk + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +type FileType byte + +const ( + FileTypeTXT FileType = 0x00 + FileTypeINT FileType = 0x01 + FileTypeAPP FileType = 0x02 + FileTypeBIN FileType = 0x04 + FileTypeS FileType = 0x08 + FileTypeREL FileType = 0x10 + FileTypeA FileType = 0x20 + FileTypeB FileType = 0x40 +) + +var AppleDOSTypeMap = map[FileType][2]string{ + 0x00: [2]string{"TXT", "ASCII Text"}, + 0x01: [2]string{"INT", "Integer Basic Program"}, + 0x02: [2]string{"BAS", "Applesoft Basic Program"}, + 0x04: [2]string{"BIN", "Binary File"}, + 0x08: [2]string{"S", "S File Type"}, + 0x10: [2]string{"REL", "Relocatable Object Code"}, + 0x20: [2]string{"A", "A File Type"}, + 0x40: [2]string{"B", "B File Type"}, +} + +func (ft FileType) String() string { + + info, ok := AppleDOSTypeMap[ft] + if ok { + return info[1] + } + + return "Unknown" +} + +func AppleDOSFileTypeFromExt(ext string) FileType { + for ft, info := range AppleDOSTypeMap { + if strings.ToUpper(ext) == info[0] { + return ft + } + } + return 0x04 +} + +func (ft FileType) Ext() string { + + info, ok := AppleDOSTypeMap[ft] + if ok { + return info[0] + } + + return "BIN" +} + +type FileDescriptor struct { + Data []byte + trackid, sectorid int + sectoroffset int +} + +func (fd *FileDescriptor) SetData(data []byte, t, s, o int) { + + fd.trackid = t + fd.sectorid = s + fd.sectoroffset = o + + if fd.Data == nil && len(data) == 35 { + fd.Data = data + } + + for i, v := range data { + fd.Data[i] = v + } +} + +func (fd *FileDescriptor) Publish(dsk *DSKWrapper) error { + + err := dsk.Seek(fd.trackid, fd.sectorid) + if err != nil { + return err + } + block := dsk.Read() + + for i, v := range fd.Data { + block[fd.sectoroffset+i] = v + } + + dsk.Write(block) + + return nil + +} + +func (fd *FileDescriptor) IsUnused() bool { + return fd.Data[0] == 0xff || fd.Type().String() == "Unknown" || fd.TotalSectors() == 0 +} + +func (fd *FileDescriptor) GetTrackSectorListStart() (int, int) { + return int(fd.Data[0]), int(fd.Data[1]) +} + +func (fd *FileDescriptor) IsLocked() bool { + return (fd.Data[2]&0x80 != 0) +} + +func (fd *FileDescriptor) SetLocked(b bool) { + fd.Data[2] = fd.Data[2] & 0x7f + if b { + fd.Data[2] = fd.Data[2] | 0x80 + } +} + +func (fd *FileDescriptor) Type() FileType { + return FileType(fd.Data[2] & 0x7f) +} + +func (fd *FileDescriptor) SetType(t FileType) { + fd.Data[0x02] = byte(t) +} + +func AsciiToPoke(b byte) byte { + if b < 32 || b > 127 { + b = 32 + } + return b | 128 +} + +func (fd *FileDescriptor) SetName(s string) { + maxlen := len(s) + if maxlen > 30 { + maxlen = 30 + } + for i := 0; i < 30; i++ { + fd.Data[0x03+i] = 0xa0 + } + for i, b := range []byte(s) { + if i >= maxlen { + break + } + fd.Data[0x03+i] = AsciiToPoke(b) + } +} + +func (fd *FileDescriptor) Name() string { + r := fd.Data[0x03:0x21] + s := "" + for _, v := range r { + ch := PokeToAscii(uint(v), false) + s = s + string(ch) + } + + s = strings.ToLower(strings.Trim(s, " ")) + + switch fd.Type() { + case FileTypeAPP: + s += ".a" + case FileTypeINT: + s += ".i" + case FileTypeBIN: + s += ".s" + case FileTypeTXT: + s += ".t" + } + + return s +} + +func (fd *FileDescriptor) NameUnadorned() string { + r := fd.Data[0x03:0x21] + s := "" + for _, v := range r { + ch := PokeToAscii(uint(v), false) + s = s + string(ch) + } + + s = strings.ToLower(strings.Trim(s, " ")) + + return s +} + +func (fd *FileDescriptor) NameBytes() []byte { + return fd.Data[0x03:0x21] +} + +func (fd *FileDescriptor) NameOK() bool { + for _, v := range fd.NameBytes() { + if v < 32 { + return false + } + } + + return true +} + +func (fd *FileDescriptor) TotalSectors() int { + return int(fd.Data[0x21]) + 256*int(fd.Data[0x22]) +} + +func (fd *FileDescriptor) SetTotalSectors(v int) { + fd.Data[0x21] = byte(v & 0xff) + fd.Data[0x22] = byte(v / 0x100) +} + +func (fd *FileDescriptor) SetTrackSectorListStart(t, s int) { + fd.Data[0] = byte(t) + fd.Data[1] = byte(s) +} + +type VTOC struct { + Data [256]byte + t, s int +} + +func (fd *VTOC) SetData(data []byte, t, s int) { + fd.t, fd.s = t, s + for i, v := range data { + fd.Data[i] = v + } +} + +func (fd *VTOC) GetCatalogStart() (int, int) { + return int(fd.Data[1]), int(fd.Data[2]) +} + +func (fd *VTOC) GetDOSVersion() byte { + return fd.Data[3] +} + +func (fd *VTOC) GetVolumeID() byte { + return fd.Data[6] +} + +func (fd *VTOC) GetMaxTSPairsPerSector() int { + return int(fd.Data[0x27]) +} + +func (fd *VTOC) GetTracks() int { + return int(fd.Data[0x34]) +} + +func (fd *VTOC) GetSectors() int { + return int(fd.Data[0x35]) +} + +func (fd *VTOC) GetTrackOrder() int { + return int(fd.Data[0x31]) +} + +func (fd *VTOC) BytesPerSector() int { + return int(fd.Data[0x36]) + 256*int(fd.Data[0x37]) +} + +func (fd *VTOC) IsTSFree(t, s int) bool { + offset := 0x38 + t*4 + if s < 8 { + offset++ + } + bitmask := byte(1 << uint(s&0x7)) + + return (fd.Data[offset]&bitmask != 0) +} + +// SetTSFree marks a T/S free or not +func (fd *VTOC) SetTSFree(t, s int, b bool) { + offset := 0x38 + t*4 + if s < 8 { + offset++ + } + bitmask := byte(1 << uint(s&0x7)) + clrmask := 0xff ^ bitmask + + v := fd.Data[offset] + if b { + v |= bitmask + } else { + v &= clrmask + } + + fd.Data[offset] = v +} + +func (fd *VTOC) Publish(dsk *DSKWrapper) error { + err := dsk.Seek(fd.t, fd.s) + if err != nil { + return err + } + + dsk.Write(fd.Data[:]) + + return nil +} + +func (fd *VTOC) DumpMap() { + + fmt.Printf("Disk has %d tracks and %d sectors per track, %d bytes per sector (ordering %d)...\n", fd.GetTracks(), fd.GetSectors(), fd.BytesPerSector(), fd.GetTrackOrder()) + fmt.Printf("Volume ID is %d\n", fd.GetVolumeID()) + ct, cs := fd.GetCatalogStart() + fmt.Printf("Catalog starts at T%d, S%d\n", ct, cs) + + tcount := fd.GetTracks() + scount := fd.GetSectors() + + for t := 0; t < tcount; t++ { + fmt.Printf("TRACK %.2x: |", t) + for s := 0; s < scount; s++ { + if fd.IsTSFree(t, s) { + fmt.Print(".") + } else { + fmt.Print("X") + } + } + fmt.Println("|") + } + +} + +func (dsk *DSKWrapper) IsAppleDOS() (bool, DiskFormat, SectorOrder) { + + oldFormat := dsk.Format + oldLayout := dsk.Layout + + defer func() { + dsk.Format = oldFormat + dsk.Layout = oldLayout + }() + + if len(dsk.Data) == STD_DISK_BYTES { + + layouts := []SectorOrder{SectorOrderDOS33, SectorOrderDOS33Alt, SectorOrderProDOS, SectorOrderProDOSLinear} + + for _, l := range layouts { + + dsk.Layout = l + + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + continue + } + + if vtoc.GetTracks() != 35 || vtoc.GetSectors() != 16 { + continue + } + + _, files, err := dsk.AppleDOSGetCatalog("*") + if err != nil { + continue + } + + if len(files) > 0 { + return true, GetDiskFormat(DF_DOS_SECTORS_16), l + } + + } + + } else if len(dsk.Data) == STD_DISK_BYTES_OLD { + + layouts := []SectorOrder{SectorOrderDOS33, SectorOrderDOS33Alt, SectorOrderProDOS, SectorOrderProDOSLinear} + + dsk.Format = GetDiskFormat(DF_DOS_SECTORS_13) + + for _, l := range layouts { + dsk.Layout = l + + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + continue + } + + if vtoc.GetTracks() != 35 || vtoc.GetSectors() != 13 { + continue + } + + _, files, err := dsk.AppleDOSGetCatalog("*") + if err != nil { + continue + } + + if len(files) > 0 { + return true, GetDiskFormat(DF_DOS_SECTORS_13), l + } + + } + + } + + return false, oldFormat, oldLayout + +} + +func (d *DSKWrapper) AppleDOSReadFileRaw(fd FileDescriptor) (int, int, []byte, error) { + + data, e := d.AppleDOSReadFileSectors(fd, -1) + + if e != nil || len(data) == 0 { + return 0, 0, data, e + } + + switch fd.Type() { + case FileTypeINT: + l := int(data[0]) + 256*int(data[1]) + if l+2 > len(data) { + l = len(data) - 2 + } + return l, 0x801, data[2 : 2+l], nil + case FileTypeAPP: + l := int(data[0]) + 256*int(data[1]) + if l+2 > len(data) { + l = len(data) - 2 + } + return l, 0x801, data[2 : 2+l], nil + case FileTypeTXT: + return len(data), 0x0000, data, nil + case FileTypeBIN: + addr := int(data[0]) + 256*int(data[1]) + l := int(data[2]) + 256*int(data[3]) + if l+4 > len(data) { + l = len(data) - 4 + } + //fmt.Printf("%x, %x, %x\n", l, addr, len(data)) + return l, addr, data[4 : 4+l], nil + default: + l := int(data[0]) + 256*int(data[1]) + if l+2 > len(data) { + l = len(data) - 2 + } + return l, 0, data[2 : 2+l], nil + } + +} + +func (d *DSKWrapper) AppleDOSGetVTOC() (*VTOC, error) { + e := d.Seek(17, 0) + if e != nil { + return nil, e + } + data := d.Read() + + vtoc := &VTOC{} + vtoc.SetData(data, 17, 0) + return vtoc, nil +} + +func (dsk *DSKWrapper) AppleDOSUsedBitmap() ([]bool, error) { + + var out []bool = make([]bool, dsk.Format.TPD()*dsk.Format.SPT()) + + _, files, err := dsk.AppleDOSGetCatalog("*") + if err != nil { + return out, err + } + + for _, f := range files { + tslist, err := dsk.AppleDOSGetFileSectors(f, 0) + if err == nil { + for _, pair := range tslist { + track := pair[0] + sector := pair[1] + out[track*dsk.Format.SPT()+sector] = true + } + } + } + + return out, nil + +} + +func (d *DSKWrapper) AppleDOSGetCatalog(pattern string) (*VTOC, []FileDescriptor, error) { + + var files []FileDescriptor + var e error + var vtoc *VTOC + + vtoc, e = d.AppleDOSGetVTOC() + if e != nil { + return vtoc, files, e + } + + count := 0 + ct, cs := vtoc.GetCatalogStart() + + e = d.Seek(ct, cs) + if e != nil { + return vtoc, files, e + } + + data := d.Read() + + var re *regexp.Regexp + if pattern != "" { + patterntmp := strings.Replace(pattern, ".", "[.]", -1) + patterntmp = strings.Replace(patterntmp, "*", ".*", -1) + patterntmp = "(?i)^" + patterntmp + "$" + re = regexp.MustCompile(patterntmp) + } + + for e == nil && count < 105 { + slot := count % 7 + pos := 0x0b + 35*slot + + fd := FileDescriptor{} + fd.SetData(data[pos:pos+35], ct, cs, pos) + + var skipname bool = false + if re != nil { + skipname = !re.MatchString(fd.Name()) + } + + if fd.Data[0] != 0xff && fd.Data[0] != 0x00 && fd.Type().String() != "Unknown" && !skipname { + files = append(files, fd) + } + count++ + if count%7 == 0 { + // move to next catalog sector + ct = int(data[1]) + cs = int(data[2]) + if ct == 0 { + return vtoc, files, nil + } + e = d.Seek(ct, cs) + if e != nil { + return vtoc, files, e + } + data = d.Read() + } + } + + return vtoc, files, nil +} + +func (d *DSKWrapper) AppleDOSGetFileSectors(fd FileDescriptor, maxblocks int) ([][2]int, error) { + var e error + var data []byte + tl, sl := fd.GetTrackSectorListStart() + + // var tracks []int + // var sectors []int + + var tslist [][2]int + + var tsmap = make(map[int]int) + + for e == nil && (tl != 0 || sl != 0) { + // Get TS List + e = d.Seek(tl, sl) + if e != nil { + return tslist, e + } + data = d.Read() + + //fmt.Printf("DEBUG: T/S List follows from T%d, S%d:\n", tl, sl) + //Dump(data) + + ptr := 0x0c + for ptr < 0x100 { + // check entry + t, s := int(data[ptr]), int(data[ptr+1]) + + if t == 0 && s == 0 || t >= d.Format.TPD() || s >= d.Format.SPT() { + //fmt.Println("BREAK ptr =", ptr, len(tracks)) + break + } + + //fmt.Printf("File block at T%d, S%d\n", t, s) + + // tracks = append(tracks, t) + // sectors = append(sectors, s) + + tslist = append(tslist, [2]int{t, s}) + + // next entry + ptr += 2 + } + + // get next TS List block + ntl, nsl := int(data[1]), int(data[2]) + if _, ex := tsmap[100*ntl+nsl]; ex { + //fmt.Printf("circular ts list") + break + } + + tl, sl = ntl, nsl + + tsmap[100*tl+sl] = 1 + + //fmt.Printf("Next Track Sector list is at T%d, S%d (%d)\n", tl, sl, len(tracks)) + + } + + return tslist, nil +} + +func (d *DSKWrapper) AppleDOSReadFileSectors(fd FileDescriptor, maxblocks int) ([]byte, error) { + var e error + var data []byte + var file []byte + tl, sl := fd.GetTrackSectorListStart() + + var tracks []int + var sectors []int + + var tsmap = make(map[int]int) + + for e == nil && (tl != 0 || sl != 0) { + // Get TS List + e = d.Seek(tl, sl) + if e != nil { + return file, e + } + data = d.Read() + + //fmt.Printf("DEBUG: T/S List follows from T%d, S%d:\n", tl, sl) + //Dump(data) + + ptr := 0x0c + for ptr < 0x100 { + // check entry + t, s := int(data[ptr]), int(data[ptr+1]) + + if t == 0 && s == 0 || t >= d.Format.TPD() || s >= d.Format.SPT() { + //fmt.Println("BREAK ptr =", ptr, len(tracks)) + break + } + + //fmt.Printf("File block at T%d, S%d\n", t, s) + + tracks = append(tracks, t) + sectors = append(sectors, s) + + // next entry + ptr += 2 + } + + // get next TS List block + ntl, nsl := int(data[1]), int(data[2]) + if _, ex := tsmap[100*ntl+nsl]; ex { + //fmt.Printf("circular ts list") + break + } + + tl, sl = ntl, nsl + + tsmap[100*tl+sl] = 1 + + //fmt.Printf("Next Track Sector list is at T%d, S%d (%d)\n", tl, sl, len(tracks)) + + } + + // Here got T/S list + //fmt.Println("READING FILE") + blocksread := 0 + for i, t := range tracks { + s := sectors[i] + + //fmt.Printf("TS Fetch #%d: Track %d, %d\n", i, t, s) + + e = d.Seek(t, s) + if e != nil { + return file, e + } + c := d.Read() + + //Dump(c) + + file = append(file, c...) + blocksread++ + + if maxblocks != -1 && blocksread >= maxblocks { + break + } + } + + return file, nil +} + +func (d *DSKWrapper) AppleDOSGetTSListSectors(fd FileDescriptor, maxblocks int) ([][2]int, error) { + var e error + var data []byte + + tl, sl := fd.GetTrackSectorListStart() + + var tslist [][2]int + + var tsmap = make(map[int]int) + + for e == nil && (tl != 0 || sl != 0) { + // Get TS List + e = d.Seek(tl, sl) + if e != nil { + return tslist, e + } + data = d.Read() + + tslist = append(tslist, [2]int{tl, sl}) + + // get next TS List block + ntl, nsl := int(data[1]), int(data[2]) + if _, ex := tsmap[100*ntl+nsl]; ex { + break + } + + tl, sl = ntl, nsl + + tsmap[100*tl+sl] = 1 + + } + + return tslist, nil +} + +func (d *DSKWrapper) AppleDOSReadFile(fd FileDescriptor) (int, int, []byte, error) { + + data, e := d.AppleDOSReadFileSectors(fd, -1) + + if e != nil { + return 0, 0, data, e + } + + switch fd.Type() { + case FileTypeINT: + l := int(data[0]) + 256*int(data[1]) + return l, 0x801, IntegerDetoks(data[2 : 2+l]), nil + case FileTypeAPP: + l := int(data[0]) + 256*int(data[1]) + return l, 0x801, ApplesoftDetoks(data[2 : 2+l]), nil + case FileTypeTXT: + return len(data), 0x0000, data, nil + case FileTypeBIN: + addr := int(data[0]) + 256*int(data[1]) + l := int(data[2]) + 256*int(data[3]) + //fmt.Printf("%x, %x, %x\n", l, addr, len(data)) + return l, addr, data[4 : 4+l], nil + default: + l := int(data[0]) + 256*int(data[1]) + return l, 0, data[2 : 2+l], nil + } + +} + +// AppleDOSGetFreeSectors tries to find free sectors for certain size file... +// Remember, we need space for the T/S list as well... +func (dsk *DSKWrapper) AppleDOSGetFreeSectors(size int) ([][2]int, [][2]int, error) { + + needed := make([][2]int, 0) + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + return nil, nil, err + } + + catTrack, _ := vtoc.GetCatalogStart() + + // needed: + // size/256 + 1 for data + // 1 for T/S list + dataBlocks := (size / 256) + 1 + tsListBlocks := (dataBlocks / vtoc.GetMaxTSPairsPerSector()) + 1 + totalBlocks := tsListBlocks + dataBlocks + + for t := dsk.Format.TPD() - 1; t >= 0; t-- { + + if t == catTrack { + continue // skip catalog track + } + + for s := dsk.Format.SPT() - 1; s >= 0; s-- { + + if len(needed) >= totalBlocks { + break + } + + if vtoc.IsTSFree(t, s) { + needed = append(needed, [2]int{t, s}) + } + } + + } + + if len(needed) >= totalBlocks { + return needed[:tsListBlocks], needed[tsListBlocks:], nil + } + + return nil, nil, errors.New("Insufficent space") + +} + +func (d *DSKWrapper) AppleDOSNextFreeCatalogEntry(name string) (*FileDescriptor, error) { + + var e error + var vtoc *VTOC + + vtoc, e = d.AppleDOSGetVTOC() + if e != nil { + return nil, e + } + + count := 0 + ct, cs := vtoc.GetCatalogStart() + + e = d.Seek(ct, cs) + if e != nil { + return nil, e + } + + data := d.Read() + + for e == nil && count < 105 { + slot := count % 7 + pos := 0x0b + 35*slot + + fd := FileDescriptor{} + fd.SetData(data[pos:pos+35], ct, cs, pos) + + if fd.IsUnused() { + return &fd, nil + } else if name != "" && strings.ToLower(fd.NameUnadorned()) == strings.ToLower(name) { + return &fd, nil + } + count++ + if count%7 == 0 { + // move to next catalog sector + ct = int(data[1]) + cs = int(data[2]) + if ct == 0 { + return nil, nil + } + e = d.Seek(ct, cs) + if e != nil { + return nil, e + } + data = d.Read() + } + } + + return nil, errors.New("No free entry") +} + +func (d *DSKWrapper) AppleDOSNamedCatalogEntry(name string) (*FileDescriptor, error) { + + var e error + var vtoc *VTOC + + vtoc, e = d.AppleDOSGetVTOC() + if e != nil { + return nil, e + } + + count := 0 + ct, cs := vtoc.GetCatalogStart() + + e = d.Seek(ct, cs) + if e != nil { + return nil, e + } + + data := d.Read() + + for e == nil && count < 105 { + slot := count % 7 + pos := 0x0b + 35*slot + + fd := FileDescriptor{} + fd.SetData(data[pos:pos+35], ct, cs, pos) + + //fmt.Printf("FILE NAME CHECK [%s] vs [%s]\n", strings.ToLower(fd.NameUnadorned()), strings.ToLower(name)) + + if name != "" && strings.ToLower(fd.NameUnadorned()) == strings.ToLower(name) { + return &fd, nil + } + count++ + if count%7 == 0 { + // move to next catalog sector + ct = int(data[1]) + cs = int(data[2]) + if ct == 0 { + return nil, errors.New("Not found") + } + e = d.Seek(ct, cs) + if e != nil { + return nil, e + } + data = d.Read() + } + } + + return nil, errors.New("Not found") +} + +func (dsk *DSKWrapper) AppleDOSWriteFile(name string, kind FileType, data []byte, loadAddr int) error { + + name = strings.ToUpper(name) + + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + return err + } + + if kind != FileTypeTXT { + header := []byte{byte(loadAddr % 256), byte(loadAddr / 256)} + data = append(header, data...) + } + + // 1st: check we have sufficient space... + tsBlocks, dataBlocks, err := dsk.AppleDOSGetFreeSectors(len(data)) + if err != nil { + return err + } + + fd, err := dsk.AppleDOSNamedCatalogEntry(name) + //fmt.Println("FD=", fd) + if err == nil { + if kind != fd.Type() { + return errors.New("File type mismatch") + } else { + // need to delete this file... + err = dsk.AppleDOSDeleteFile(name) + if err != nil { + return err + } + } + } else { + fd, err = dsk.AppleDOSNextFreeCatalogEntry(name) + if err != nil { + return err + } + } + + // 2nd: check we can get a free catalog entry + + // 3rd: Write the datablocks + var block int = 0 + for len(data) > 0 { + + max := STD_BYTES_PER_SECTOR + if len(data) < STD_BYTES_PER_SECTOR { + max = len(data) + } + chunk := data[:max] + // Pad final sector with 0x00 bytes + for len(chunk) < STD_BYTES_PER_SECTOR { + chunk = append(chunk, 0x00) + } + data = data[max:] + + pair := dataBlocks[block] + + track, sector := pair[0], pair[1] + + err = dsk.Seek(track, sector) + if err != nil { + return err + } + dsk.Write(chunk) + + block++ + + } + + // 4th: Write the T/S List + offset := 0 + for blockIdx, block := range tsBlocks { + listTrack, listSector := block[0], block[1] + nextTrack, nextSector := 0, 0 + if blockIdx < len(tsBlocks)-1 { + nextTrack, nextSector = tsBlocks[blockIdx+1][0], tsBlocks[blockIdx+1][1] + } + + buffer := make([]byte, STD_BYTES_PER_SECTOR) + + // header + buffer[0x01] = byte(nextTrack) + buffer[0x02] = byte(nextSector) + buffer[0x05] = byte(offset & 0xff) + buffer[0x06] = byte(offset / 0x100) + + // + count := vtoc.GetMaxTSPairsPerSector() + if offset+count >= len(dataBlocks) { + count = len(dataBlocks) - offset + } + + for i := 0; i < count; i++ { + pos := 0x0c + i*2 + buffer[pos+0x00] = byte(dataBlocks[offset+i][0]) + buffer[pos+0x01] = byte(dataBlocks[offset+i][1]) + vtoc.SetTSFree(dataBlocks[offset+i][0], dataBlocks[offset+i][1], false) + } + + // Write the sector + err = dsk.Seek(listTrack, listSector) + if err != nil { + return err + } + dsk.Write(buffer) + vtoc.SetTSFree(listTrack, listSector, false) + } + + err = vtoc.Publish(dsk) + if err != nil { + return err + } + + // 5th and finally: Let's make that catalog entry + fd.SetName(name) + fd.SetTrackSectorListStart(tsBlocks[0][0], tsBlocks[0][1]) + fd.SetType(kind) + fd.SetTotalSectors(len(dataBlocks)) + + return nil + +} + +func (d *DSKWrapper) AppleDOSRemoveFile(fd *FileDescriptor) error { + + vtoc, err := d.AppleDOSGetVTOC() + if err != nil { + return err + } + + if fd.IsUnused() { + return errors.New("File does not exist") + } + + tsBlocks, e := d.AppleDOSGetTSListSectors(*fd, -1) + if e != nil { + return e + } + + dataBlocks, e := d.AppleDOSGetFileSectors(*fd, -1) + if e != nil { + return e + } + + for _, pair := range dataBlocks { + vtoc.SetTSFree(pair[0], pair[1], true) + } + + for _, pair := range tsBlocks { + vtoc.SetTSFree(pair[0], pair[1], true) + } + + fd.Data[0x00] = 0xff + fd.SetName("") + return fd.Publish(d) + +} + +func (dsk *DSKWrapper) AppleDOSDeleteFile(name string) error { + + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + return err + } + + // We cheat here a bit and use the get first free entry call with + // autogrow turned off. + fd, err := dsk.AppleDOSNamedCatalogEntry(name) + if err != nil { + return err + } + + if fd.IsUnused() { + return errors.New("Not found") + } + + // At this stage we have a match so get blocks to remove + tsBlocks, e := dsk.AppleDOSGetTSListSectors(*fd, -1) + if e != nil { + return e + } + + dataBlocks, e := dsk.AppleDOSGetFileSectors(*fd, -1) + if e != nil { + return e + } + + for _, pair := range dataBlocks { + vtoc.SetTSFree(pair[0], pair[1], true) + } + + for _, pair := range tsBlocks { + vtoc.SetTSFree(pair[0], pair[1], true) + } + + err = vtoc.Publish(dsk) + if err != nil { + return err + } + + fd.Data[0x00] = 0xff + fd.SetName("") + return fd.Publish(dsk) + +} + +func (dsk *DSKWrapper) AppleDOSSetLocked(name string, lock bool) error { + + // We cheat here a bit and use the get first free entry call with + // autogrow turned off. + fd, err := dsk.AppleDOSNamedCatalogEntry(name) + if err != nil { + return err + } + + if fd.IsUnused() { + return errors.New("Not found") + } + + fd.SetLocked(lock) + return fd.Publish(dsk) + +} + +func (dsk *DSKWrapper) AppleDOSRenameFile(name, newname string) error { + + fd, err := dsk.AppleDOSNamedCatalogEntry(name) + if err != nil { + return err + } + + _, err = dsk.AppleDOSNamedCatalogEntry(newname) + if err == nil { + return errors.New("New name already exists") + } + + // can rename here + fd.SetName(newname) + return fd.Publish(dsk) + +} diff --git a/disk/diskimagepas.go b/disk/diskimagepas.go new file mode 100644 index 0000000..78f30b7 --- /dev/null +++ b/disk/diskimagepas.go @@ -0,0 +1,314 @@ +package disk + +import ( + "errors" + "regexp" + "strings" +) + +const PASCAL_BLOCK_SIZE = 512 +const PASCAL_VOLUME_BLOCK = 2 +const PASCAL_MAX_VOLUME_NAME = 7 +const PASCAL_DIRECTORY_ENTRY_LENGTH = 26 +const PASCAL_OVERSIZE_DIR = 32 + +func (dsk *DSKWrapper) IsPascal() (bool, string) { + + dsk.Format = GetDiskFormat(DF_PRODOS) + + data, err := dsk.PRODOSGetBlock(PASCAL_VOLUME_BLOCK) + if err != nil { + return false, "" + } + + if !(data[0x00] == 0 && data[0x01] == 0) || + !(data[0x04] == 0 && data[0x05] == 0) || + !(data[0x06] > 0 && data[0x06] <= PASCAL_MAX_VOLUME_NAME) { + return false, "" + } + + l := int(data[0x06]) + name := data[0x07 : 0x07+l] + + str := "" + for _, ch := range name { + if ch == 0x00 { + break + } + if ch < 0x20 || ch >= 0x7f { + return false, "" + } + + if strings.Contains("$=?,[#:", string(ch)) { + return false, "" + } + + str += string(ch) + } + + return true, str + +} + +type PascalVolumeHeader struct { + data [PASCAL_DIRECTORY_ENTRY_LENGTH]byte +} + +func (pvh *PascalVolumeHeader) SetData(data []byte) { + for i, v := range data { + if i < len(pvh.data) { + pvh.data[i] = v + } + } +} + +func (pvh *PascalVolumeHeader) GetStartBlock() int { + return int(pvh.data[0x00]) + 256*int(pvh.data[0x01]) +} + +func (pvh *PascalVolumeHeader) GetNextBlock() int { + return int(pvh.data[0x02]) + 256*int(pvh.data[0x03]) +} + +type PascalFileType int + +const ( + FileType_PAS_NONE PascalFileType = 0 + FileType_PAS_BADD PascalFileType = 1 + FileType_PAS_CODE PascalFileType = 2 + FileType_PAS_TEXT PascalFileType = 3 + FileType_PAS_INFO PascalFileType = 4 + FileType_PAS_DATA PascalFileType = 5 + FileType_PAS_GRAF PascalFileType = 6 + FileType_PAS_FOTO PascalFileType = 7 + FileType_PAS_SECD PascalFileType = 8 +) + +var PascalTypeMap = map[PascalFileType][2]string{ + 0x00: [2]string{"UNK", "ASCII Text"}, + 0x01: [2]string{"BAD", "Bad Block"}, + 0x02: [2]string{"PCD", "Pascal Code"}, + 0x03: [2]string{"PTX", "Pascal Text"}, + 0x04: [2]string{"PIF", "Pascal Info"}, + 0x05: [2]string{"PDA", "Pascal Data"}, + 0x06: [2]string{"GRF", "Pascal Graphics"}, + 0x07: [2]string{"FOT", "HiRes Graphics"}, + 0x08: [2]string{"SEC", "Secure Directory"}, +} + +func (ft PascalFileType) String() string { + + info, ok := PascalTypeMap[ft] + if ok { + return info[1] + } + + return "Unknown" + +} + +func (ft PascalFileType) Ext() string { + + info, ok := PascalTypeMap[ft] + if ok { + return info[0] + } + + return "UNK" + +} + +func PascalFileTypeFromExt(ext string) PascalFileType { + for ft, info := range PascalTypeMap { + if strings.ToUpper(ext) == info[0] { + return ft + } + } + return 0x00 +} + +func (pvh *PascalVolumeHeader) GetType() int { + return int(int(pvh.data[0x04]) + 256*int(pvh.data[0x05])) +} + +func (pvh *PascalVolumeHeader) GetNameLength() int { + return int(pvh.data[0x06]) & 0x07 +} + +func (pvh *PascalVolumeHeader) GetName() string { + l := pvh.GetNameLength() + return string(pvh.data[0x07 : 0x07+l]) +} + +func (pvh *PascalVolumeHeader) GetTotalBlocks() int { + return int(pvh.data[0x0e]) + 256*int(pvh.data[0x0f]) +} + +func (pvh *PascalVolumeHeader) GetNumFiles() int { + return int(pvh.data[0x10]) + 256*int(pvh.data[0x11]) +} + +type PascalFileEntry struct { + data [PASCAL_DIRECTORY_ENTRY_LENGTH]byte +} + +func (pfe *PascalFileEntry) SetData(data []byte) { + for i, v := range data { + if i < len(pfe.data) { + pfe.data[i] = v + } + } +} + +func (pvh *PascalFileEntry) IsLocked() bool { + return true +} + +func (pvh *PascalFileEntry) GetStartBlock() int { + return int(pvh.data[0x00]) + 256*int(pvh.data[0x01]) +} + +func (pvh *PascalFileEntry) GetNextBlock() int { + return int(pvh.data[0x02]) + 256*int(pvh.data[0x03]) +} + +func (pvh *PascalFileEntry) GetType() PascalFileType { + return PascalFileType(int(pvh.data[0x04]) + 256*int(pvh.data[0x05])) +} + +func (pvh *PascalFileEntry) GetNameLength() int { + return int(pvh.data[0x06]) & 0x0f +} + +func (pvh *PascalFileEntry) GetName() string { + l := pvh.GetNameLength() + return string(pvh.data[0x07 : 0x07+l]) +} + +func (pvh *PascalFileEntry) GetBytesRemaining() int { + return int(pvh.data[0x16]) + 256*int(pvh.data[0x17]) +} + +func (pvh *PascalFileEntry) GetFileSize() int { + return pvh.GetBytesRemaining() + (pvh.GetNextBlock()-pvh.GetStartBlock()-1)*PASCAL_BLOCK_SIZE +} + +func (dsk *DSKWrapper) PascalGetCatalog(pattern string) ([]*PascalFileEntry, error) { + + pattern = strings.Replace(pattern, ".", "[.]", -1) + pattern = strings.Replace(pattern, "*", ".*", -1) + pattern = strings.Replace(pattern, "?", ".", -1) + + rx := regexp.MustCompile("(?i)" + pattern) + + files := make([]*PascalFileEntry, 0) + + // + + d, err := dsk.PRODOSGetBlock(PASCAL_VOLUME_BLOCK) + if err != nil { + return nil, err + } + + pvh := &PascalVolumeHeader{} + pvh.SetData(d) + numBlocks := pvh.GetNextBlock() - PASCAL_VOLUME_BLOCK + + if numBlocks < 0 || numBlocks > PASCAL_OVERSIZE_DIR { + return files, errors.New("Directory appears corrupt") + } + + // disk catalog is okay + catdata := make([]byte, 0) + for block := PASCAL_VOLUME_BLOCK; block < PASCAL_VOLUME_BLOCK+numBlocks; block++ { + data, err := dsk.PRODOSGetBlock(block) + if err != nil { + return files, err + } + catdata = append(catdata, data...) + } + + dirPtr := PASCAL_DIRECTORY_ENTRY_LENGTH + for i := 0; i < pvh.GetNumFiles(); i++ { + b := catdata[dirPtr : dirPtr+PASCAL_DIRECTORY_ENTRY_LENGTH] + fd := &PascalFileEntry{} + fd.SetData(b) + // add file + + if rx.MatchString(fd.GetName()) { + + files = append(files, fd) + + } + + // move + dirPtr += PASCAL_DIRECTORY_ENTRY_LENGTH + } + + return files, nil + +} + +func (dsk *DSKWrapper) PascalUsedBitmap() ([]bool, error) { + + activeBlocks := dsk.Format.BPD() + + used := make([]bool, activeBlocks) + + files, err := dsk.PascalGetCatalog("*") + if err != nil { + return used, err + } + + for _, file := range files { + + length := file.GetNextBlock() - file.GetStartBlock() + start := file.GetStartBlock() + if start+length > activeBlocks { + continue // file is bad + } + + for block := start; block < start+length; block++ { + used[block] = true + } + + } + + return used, nil + +} + +func (dsk *DSKWrapper) PascalReadFile(file *PascalFileEntry) ([]byte, error) { + + activeSectors := dsk.Format.BPD() + + length := file.GetNextBlock() - file.GetStartBlock() + start := file.GetStartBlock() + + // If file is damaged return nothing + if start+length > activeSectors { + return []byte(nil), nil + } + + block := start + data := make([]byte, 0) + for block < start+length && len(data) < file.GetFileSize() { + + chunk, err := dsk.PRODOSGetBlock(block) + if err != nil { + return data, err + } + needed := file.GetFileSize() - len(data) + if needed >= PASCAL_BLOCK_SIZE { + data = append(data, chunk...) + } else { + data = append(data, chunk[:needed]...) + } + + block++ + + } + + return data, nil + +} diff --git a/disk/diskimagepd.go b/disk/diskimagepd.go new file mode 100644 index 0000000..6ba51c7 --- /dev/null +++ b/disk/diskimagepd.go @@ -0,0 +1,1852 @@ +package disk + +import ( + "errors" + "fmt" + "regexp" + "strings" + "time" +) + +type VDH struct { + Data []byte + blockid int + blockoffset int +} + +func (fd *VDH) CreateTime() time.Time { + + b := fd.Data[0x1C:0x20] + + return prodosStampBytesToTime(b) + +} + +func (fd *VDH) SetCreateTime(t time.Time) { + + b := timeToProdosStampBytes(t) + for i, v := range b { + fd.Data[0x1C+i] = v + } + +} + +func (fd *VDH) SetData(data []byte, blockid, blockoffset int) { + + fd.blockid = blockid + fd.blockoffset = blockoffset + + if fd.Data == nil && len(data) == 39 { + fd.Data = data + //Println("VDH: ") + //Dump(data) + } + + for i, v := range data { + fd.Data[i] = v + } +} + +func (fd *VDH) SetName(name string) { + + name = strings.ToUpper(name) + + if len(name) > 15 { + name = name[:15] + } + + l := len(name) + + for i, v := range []byte(name) { + fd.Data[1+i] = v + } + + fd.Data[0] = (fd.Data[0] & 0xf0) | byte(l) + +} + +func (fd *VDH) SetStorageType(t ProDOSStorageType) { + + fd.Data[0] = (fd.Data[0] & 0x0f) | (byte(t) << 4) + +} + +func (fd *VDH) GetNameLength() int { + return int(fd.Data[0] & 0xf) +} + +func (fd *VDH) GetStorageType() ProDOSStorageType { + return ProDOSStorageType((fd.Data[0]) >> 4) +} + +func (fd *VDH) GetDirName() string { + return fd.GetVolumeName() +} + +func (fd *VDH) GetVolumeName() string { + + l := fd.GetNameLength() + + b := fd.Data[1 : 1+l] + + s := "" + for _, v := range b { + s += string(rune(PokeToAscii(uint(v), false))) + } + + return strings.Trim(s, " ") + +} + +func (fd *VDH) GetVersion() int { + return int(fd.Data[28]) +} + +func (fd *VDH) SetVersion(b int) { + fd.Data[28] = byte(b) +} + +func (fd *VDH) GetMinVersion() int { + return int(fd.Data[29]) +} + +func (fd *VDH) SetMinVersion(b int) { + fd.Data[29] = byte(b) +} + +func (fd *VDH) GetAccess() ProDOSAccessMode { + return ProDOSAccessMode(fd.Data[30]) +} + +func (fd *VDH) SetAccess(m ProDOSAccessMode) { + fd.Data[30] = byte(m) +} + +func (fd *VDH) GetEntryLength() int { + return int(fd.Data[31]) +} + +func (fd *VDH) SetEntryLength(b int) { + fd.Data[31] = byte(b & 0xff) +} + +func (fd *VDH) GetEntriesPerBlock() int { + return int(fd.Data[32]) +} + +func (fd *VDH) SetEntriesPerBlock(b int) { + fd.Data[32] = byte(b & 0xff) +} + +func (fd *VDH) GetFileCount() int { + return int(fd.Data[33]) + 256*int(fd.Data[34]) +} + +func (fd *VDH) SetFileCount(c int) { + fd.Data[33] = byte(c & 0xff) + fd.Data[34] = byte(c / 0x100) +} + +func (fd *VDH) GetBitmapPointer() int { + return int(fd.Data[35]) + 256*int(fd.Data[36]) +} + +func (fd *VDH) GetTotalBlocks() int { + return int(fd.Data[37]) + 256*int(fd.Data[38]) +} + +func (fd *VDH) SetTotalBlocks(b int) { + fd.Data[37] = byte(b % 256) + fd.Data[38] = byte(b / 256) +} + +func (fd *VDH) GetDirParentPointer() int { + return int(fd.Data[35]) + 256*int(fd.Data[35]) +} + +func (fd *VDH) SetDirParentPointer(b int) { + fd.Data[35] = byte(b & 0xff) + fd.Data[36] = byte(b / 0x100) +} + +func (fd *VDH) GetDirParentEntry() int { + return int(fd.Data[37]) +} + +func (fd *VDH) SetDirParentEntry(b int) { + fd.Data[37] = byte(b & 0xff) +} + +func (fd *VDH) GetDirParentEntryLength() int { + return int(fd.Data[38]) +} + +func (fd *VDH) SetDirParentEntryLength(b int) { + fd.Data[38] = byte(b & 0xff) +} + +func (fd *VDH) Publish(dsk *DSKWrapper) error { + bd, err := dsk.PRODOSGetBlock(fd.blockid) + if err != nil { + return err + } + for i, v := range fd.Data { + bd[fd.blockoffset+i] = v + } + fmt.Printf("Writing dir header at block %d\n", fd.blockid) + + fmt.Printf("Data=%v\n", bd) + + return dsk.PRODOSWrite(fd.blockid, bd) +} + +func (dsk *DSKWrapper) IsProDOS() (bool, DiskFormat, SectorOrder) { + + oldFormat := dsk.Format + oldLayout := dsk.Layout + + defer func() { + dsk.Format = oldFormat + dsk.Layout = oldLayout + }() + + if len(dsk.Data) == STD_DISK_BYTES { + + layouts := []SectorOrder{SectorOrderDOS33, SectorOrderDOS33Alt, SectorOrderProDOS, SectorOrderProDOSLinear} + + for _, l := range layouts { + + dsk.Layout = l + vdh, err := dsk.PRODOSGetVDH(2) + if err != nil { + return false, oldFormat, oldLayout + } + + if vdh.GetTotalBlocks() == 280 && vdh.GetStorageType() == 0xf { + return true, GetDiskFormat(DF_PRODOS), l + } + + } + + } else if len(dsk.Data) == PRODOS_800KB_DISK_BYTES { + + layouts := []SectorOrder{SectorOrderDOS33, SectorOrderDOS33Alt, SectorOrderProDOS} + + for _, l := range layouts { + + dsk.Layout = l + vdh, err := dsk.PRODOSGetVDH(2) + if err != nil { + return false, oldFormat, oldLayout + } + + if vdh.GetTotalBlocks() == 1600 && vdh.GetStorageType() == 0xf { + return true, GetDiskFormat(DF_PRODOS_800KB), l + } + + } + + } + + return false, oldFormat, oldLayout + +} + +func (dsk *DSKWrapper) PRODOS800GetVolumeBitmap() (ProDOSVolumeBitmap, error) { + + var vb ProDOSVolumeBitmap + + vdh, err := dsk.PRODOS800GetVDH(2) + if err != nil { + return vb, err + } + + b := vdh.GetBitmapPointer() + + data, err := dsk.PRODOS800GetBlock(b) + if err != nil { + return vb, err + } + + //copy(vb[:], data) + + vb = ProDOSVolumeBitmap{ + Data: data, + blockid: b, + blockoffset: 0, + } + + return vb, nil + +} + +func (dsk *DSKWrapper) PRODOSGetVolumeBitmap() (ProDOSVolumeBitmap, error) { + + var vb ProDOSVolumeBitmap + + vdh, err := dsk.PRODOSGetVDH(2) + if err != nil { + return vb, err + } + + b := vdh.GetBitmapPointer() + + data, err := dsk.PRODOSGetBlock(b) + if err != nil { + return vb, err + } + + //copy(vb[:], data) + vb = ProDOSVolumeBitmap{ + Data: data, + blockid: b, + blockoffset: 0, + } + + return vb, nil + +} + +type ProDOSVolumeBitmap struct { + Data []byte + blockid int + blockoffset int +} + +func (vb ProDOSVolumeBitmap) IsBlockFree(b int) bool { + + bidx := b / 8 + bit := 7 - (b % 8) + mask := byte(1 << uint(bit)) + + return (vb.Data[bidx] & mask) == mask + +} + +func (vb ProDOSVolumeBitmap) SetBlockFree(b int, free bool) { + bidx := b / 8 + bit := 7 - (b % 8) + setmask := byte(1 << uint(bit)) + clrmask := 0xff ^ setmask + + if free { + vb.Data[bidx] = vb.Data[bidx] | setmask + } else { + vb.Data[bidx] = vb.Data[bidx] & clrmask + } +} + +// -- Prodos file descriptor +type ProDOSFileDescriptor struct { + Data []byte + blockid int + blockoffset int + entryNum int +} + +func (fd *ProDOSFileDescriptor) SetData(data []byte, blockid int, blockoffset int) { + + fd.blockid = blockid + fd.blockoffset = blockoffset + + if fd.Data == nil && len(data) == 39 { + fd.Data = data + return + } + + for i, v := range data { + fd.Data[i] = v + } +} + +func (fd *ProDOSFileDescriptor) GetNameLength() int { + return int(fd.Data[0] & 0xf) +} + +func (fd *ProDOSFileDescriptor) GetStorageType() ProDOSStorageType { + return ProDOSStorageType((fd.Data[0]) >> 4) +} + +func (fd *ProDOSFileDescriptor) Name() string { + + l := fd.GetNameLength() + + b := fd.Data[1 : 1+l] + + s := "" + for _, v := range b { + s += string(rune(PokeToAscii(uint(v), false))) + } + + s = strings.ToLower(strings.Trim(s, " ")) + + switch fd.Type() { + case FileType_PD_APP: + s += ".a" + case FileType_PD_INT: + s += ".i" + case FileType_PD_BIN: + s += ".s" + case FileType_PD_TXT: + s += ".t" + case FileType_PD_SYS: + s += ".s" + } + + return s + +} + +func (fd *ProDOSFileDescriptor) NameUnadorned() string { + + l := fd.GetNameLength() + + b := fd.Data[1 : 1+l] + + s := "" + for _, v := range b { + s += string(rune(PokeToAscii(uint(v), false))) + } + + s = strings.ToLower(strings.Trim(s, " ")) + + return s + +} + +func (fd *ProDOSFileDescriptor) SetName(name string) { + + name = strings.ToUpper(name) + + if len(name) > 15 { + name = name[:15] + } + + l := len(name) + + for i, v := range []byte(name) { + fd.Data[1+i] = v + } + + fd.Data[0] = (fd.Data[0] & 0xf0) | byte(l) + +} + +func (fd *ProDOSFileDescriptor) SetStorageType(t ProDOSStorageType) { + + fd.Data[0] = (fd.Data[0] & 0x0f) | (byte(t) << 4) + +} + +func (fd *ProDOSFileDescriptor) SetAccessMode(t ProDOSAccessMode) { + + fd.Data[0x1e] = byte(t) + +} + +func (fd *ProDOSFileDescriptor) AccessMode() ProDOSAccessMode { + return ProDOSAccessMode(fd.Data[0x1e]) +} + +func (fd *ProDOSFileDescriptor) SetLocked(b bool) { + accessMode := fd.AccessMode() + if b { + accessMode = accessMode & (AccessType_Changed | AccessType_Readable) + } else { + accessMode = accessMode | AccessType_Destroy | AccessType_Writable | AccessType_Rename + } + fd.SetAccessMode(accessMode) +} + +func (fd *ProDOSFileDescriptor) IsLocked() bool { + a := fd.AccessMode() + return a&(AccessType_Destroy|AccessType_Rename|AccessType_Writable) == 0 +} + +func prodosStampBytesToTime(in []byte) time.Time { + dbits := (int(in[0x01]) << 8) | int(in[0x00]) + day := dbits & 31 + month := (dbits >> 5) & 15 + year := (dbits >> 9) & 127 + tbits := (int(in[0x03]) << 8) | int(in[0x02]) + mins := tbits & 63 + hours := (tbits >> 8) + + if year < 70 { + year += 100 + } + year += 1900 + + return time.Date(year, time.Month(month), day, hours, mins, 0, 0, time.Local) +} + +func timeToProdosStampBytes(t time.Time) []byte { + year, month, day, hour, minute := t.Year(), int(t.Month()), t.Day(), t.Hour(), t.Minute() + year = year - 1900 + if year > 99 { + year -= 100 + } + + dbits := (year << 9) | (month << 5) | day + tbits := (hour << 8) | minute + + return []byte{ + byte(dbits & 0xff), + byte(dbits >> 8), + byte(tbits & 0xff), + byte(tbits >> 8), + } + +} + +func (fd *ProDOSFileDescriptor) CreateTime() time.Time { + + b := fd.Data[0x18:0x1C] + + return prodosStampBytesToTime(b) + +} + +func (fd *ProDOSFileDescriptor) SetCreateTime(t time.Time) { + + b := timeToProdosStampBytes(t) + for i, v := range b { + fd.Data[0x18+i] = v + } + +} + +func (fd *ProDOSFileDescriptor) ModTime() time.Time { + + b := fd.Data[0x21:0x25] + + return prodosStampBytesToTime(b) + +} + +func (fd *ProDOSFileDescriptor) SetModTime(t time.Time) { + + b := timeToProdosStampBytes(t) + for i, v := range b { + fd.Data[0x21+i] = v + } + +} + +func (fd *ProDOSFileDescriptor) GetVersion() int { + return int(fd.Data[0x1c]) +} + +func (fd *ProDOSFileDescriptor) SetVersion(b int) { + fd.Data[0x1c] = byte(b) +} + +func (fd *ProDOSFileDescriptor) GetMinVersion() int { + return int(fd.Data[0x1d]) +} + +func (fd *ProDOSFileDescriptor) SetMinVersion(b int) { + fd.Data[0x1d] = byte(b) +} + +type ProDOSAccessMode byte + +const ( + AccessType_Destroy ProDOSAccessMode = 0x80 + AccessType_Rename ProDOSAccessMode = 0x40 + AccessType_Changed ProDOSAccessMode = 0x20 + AccessType_Writable ProDOSAccessMode = 0x02 + AccessType_Readable ProDOSAccessMode = 0x01 + // + AccessType_Default ProDOSAccessMode = AccessType_Readable | AccessType_Writable | AccessType_Rename | AccessType_Destroy +) + +type ProDOSStorageType byte + +const ( + StorageType_Inactive ProDOSStorageType = 0x0 + StorageType_Seedling ProDOSStorageType = 0x1 + StorageType_Sapling ProDOSStorageType = 0x2 + StorageType_Tree ProDOSStorageType = 0x3 + StorageType_SubDir_File ProDOSStorageType = 0xd + StorageType_SubDir_Header ProDOSStorageType = 0xe + StorageType_Volume_Header ProDOSStorageType = 0xf +) + +type ProDOSFileType byte + +const ( + FileType_PD_None ProDOSFileType = 0x00 + FileType_PD_TXT ProDOSFileType = 0x04 + FileType_PD_Directory ProDOSFileType = 0x0f + FileType_PD_BIN ProDOSFileType = 0x06 + FileType_PD_INT ProDOSFileType = 0xfa + FileType_PD_INT_Var ProDOSFileType = 0xfb + FileType_PD_APP ProDOSFileType = 0xfc + FileType_PD_APP_Var ProDOSFileType = 0xfd + FileType_PD_Reloc ProDOSFileType = 0xfe + FileType_PD_SYS ProDOSFileType = 0xff +) + +var ProDOSTypeMap = map[ProDOSFileType][2]string{ + 0x00: [2]string{"UNK", "Unknown"}, + 0x01: [2]string{"BAD", "Bad Block"}, + 0x02: [2]string{"PCD", "Pascal Code"}, + 0x03: [2]string{"PTX", "Pascal Text"}, + 0x04: [2]string{"TXT", "ASCII Text"}, + 0x05: [2]string{"PDA", "Pascal Data"}, + 0x06: [2]string{"BIN", "Binary File"}, + 0x07: [2]string{"FNT", "Apple III Font"}, + 0x08: [2]string{"FOT", "HiRes/Double HiRes Graphics"}, + 0x09: [2]string{"BA3", "Apple III Basic Program"}, + 0x0A: [2]string{"DA3", "Apple III Basic Data"}, + 0x0B: [2]string{"WPF", "Generic Word Processing"}, + 0x0C: [2]string{"SOS", "SOS System File"}, + 0x0F: [2]string{"DIR", "ProDOS Directory"}, + 0x10: [2]string{"RPD", "RPS Data"}, + 0x11: [2]string{"RPI", "RPS Index"}, + 0x12: [2]string{"AFD", "AppleFile Discard"}, + 0x13: [2]string{"AFM", "AppleFile Model"}, + 0x14: [2]string{"AFR", "AppleFile Report"}, + 0x15: [2]string{"SCL", "Screen Library"}, + 0x16: [2]string{"PFS", "PFS Document"}, + 0x19: [2]string{"ADB", "AppleWorks Database"}, + 0x1A: [2]string{"AWP", "AppleWorks Word Processing"}, + 0x1B: [2]string{"ASP", "AppleWorks Spreadsheet"}, + 0x80: [2]string{"GES", "System File"}, + 0x81: [2]string{"GEA", "Desk Accessory"}, + 0x82: [2]string{"GEO", "Application"}, + 0x83: [2]string{"GED", "Document"}, + 0x84: [2]string{"GEF", "Font"}, + 0x85: [2]string{"GEP", "Printer Driver"}, + 0x86: [2]string{"GEI", "Input Driver"}, + 0x87: [2]string{"GEX", "Auxiliary Driver"}, + 0x89: [2]string{"GEV", "Swap File"}, + 0x8B: [2]string{"GEC", "Clock Driver"}, + 0x8C: [2]string{"GEK", "Interface Card Driver"}, + 0x8D: [2]string{"GEW", "Formatting Data"}, + 0xA0: [2]string{"WP", " WordPerfect"}, + 0xAB: [2]string{"GSB", "Apple IIgs BASIC Program"}, + 0xAC: [2]string{"TDF", "Apple IIgs BASIC TDF"}, + 0xAD: [2]string{"BDF", "Apple IIgs BASIC Data"}, + 0x60: [2]string{"PRE", "PC Pre-Boot"}, + 0x6B: [2]string{"BIO", "PC BIOS"}, + 0x66: [2]string{"NCF", "ProDOS File Navigator Command File"}, + 0x6D: [2]string{"DVR", "PC Driver"}, + 0x6E: [2]string{"PRE", "PC Pre-Boot"}, + 0x6F: [2]string{"HDV", "PC Hard Disk Image"}, + 0x50: [2]string{"GWP", "Apple IIgs Word Processing"}, + 0x51: [2]string{"GSS", "Apple IIgs Spreadsheet"}, + 0x52: [2]string{"GDB", "Apple IIgs Database"}, + 0x53: [2]string{"DRW", "Object Oriented Graphics"}, + 0x54: [2]string{"GDP", "Apple IIgs Desktop Publishing"}, + 0x55: [2]string{"HMD", "HyperMedia"}, + 0x56: [2]string{"EDU", "Educational Program Data"}, + 0x57: [2]string{"STN", "Stationery"}, + 0x58: [2]string{"HLP", "Help File"}, + 0x59: [2]string{"COM", "Communications"}, + 0x5A: [2]string{"CFG", "Configuration"}, + 0x5B: [2]string{"ANM", "Animation"}, + 0x5C: [2]string{"MUM", "Multimedia"}, + 0x5D: [2]string{"ENT", "Entertainment"}, + 0x5E: [2]string{"DVU", "Development Utility"}, + 0x41: [2]string{"OCR", "Optical Character Recognition"}, + 0x42: [2]string{"FTD", "File Type Definitions"}, + 0x20: [2]string{"TDM", "Desktop Manager File"}, + 0x21: [2]string{"IPS", "Instant Pascal Source"}, + 0x22: [2]string{"UPV", "UCSD Pascal Volume"}, + 0x29: [2]string{"3SD", "SOS Directory"}, + 0x2A: [2]string{"8SC", "Source Code"}, + 0x2B: [2]string{"8OB", "Object Code"}, + 0x2C: [2]string{"8IC", "Interpreted Code"}, + 0x2D: [2]string{"8LD", "Language Data"}, + 0x2E: [2]string{"P8C", "ProDOS 8 Code Module"}, + 0xB0: [2]string{"SRC", "Apple IIgs Source Code"}, + 0xB1: [2]string{"OBJ", "Apple IIgs Object Code"}, + 0xB2: [2]string{"LIB", "Apple IIgs Library"}, + 0xB3: [2]string{"S16", "Apple IIgs Application Program"}, + 0xB4: [2]string{"RTL", "Apple IIgs Runtime Library"}, + 0xB5: [2]string{"EXE", "Apple IIgs Shell Script"}, + 0xB6: [2]string{"PIF", "Apple IIgs Permanent INIT"}, + 0xB7: [2]string{"TIF", "Apple IIgs Temporary INIT"}, + 0xB8: [2]string{"NDA", "Apple IIgs New Desk Accessory"}, + 0xB9: [2]string{"CDA", "Apple IIgs Classic Desk Accessory"}, + 0xBA: [2]string{"TOL", "Apple IIgs Tool"}, + 0xBB: [2]string{"DRV", "Apple IIgs Device Driver"}, + 0xBC: [2]string{"LDF", "Apple IIgs Generic Load File"}, + 0xBD: [2]string{"FST", "Apple IIgs File System Translator"}, + 0xBF: [2]string{"DOC", "Apple IIgs Document "}, + 0xC0: [2]string{"PNT", "Apple IIgs Packed Super HiRes"}, + 0xC1: [2]string{"PIC", "Apple IIgs Super HiRes"}, + 0xC2: [2]string{"ANI", "PaintWorks Animation"}, + 0xC3: [2]string{"PAL", "PaintWorks Palette"}, + 0xC5: [2]string{"OOG", "Object-Oriented Graphics"}, + 0xC6: [2]string{"SCR", "Script"}, + 0xC7: [2]string{"CDV", "Apple IIgs Control Panel"}, + 0xC8: [2]string{"FON", "Apple IIgs Font"}, + 0xC9: [2]string{"FND", "Apple IIgs Finder Data"}, + 0xCA: [2]string{"ICN", "Apple IIgs Icon "}, + 0xD5: [2]string{"MUS", "Music"}, + 0xD6: [2]string{"INS", "Instrument"}, + 0xD7: [2]string{"MDI", "MIDI"}, + 0xD8: [2]string{"SND", "Apple IIgs Audio"}, + 0xDB: [2]string{"DBM", "DB Master Document"}, + 0xE0: [2]string{"LBR", "Archive"}, + 0xE2: [2]string{"ATK", "AppleTalk Data"}, + 0xEE: [2]string{"R16", "EDASM 816 Relocatable Code"}, + 0xEF: [2]string{"PAR", "Pascal Area"}, + 0xF0: [2]string{"CMD", "ProDOS Command File"}, + 0xF1: [2]string{"OVL", "User Defined 1"}, + 0xF2: [2]string{"UD2", "User Defined 2"}, + 0xF3: [2]string{"UD3", "User Defined 3"}, + 0xF4: [2]string{"UD4", "User Defined 4"}, + 0xF5: [2]string{"BAT", "User Defined 5"}, + 0xF6: [2]string{"UD6", "User Defined 6"}, + 0xF7: [2]string{"UD7", "User Defined 7"}, + 0xF8: [2]string{"PRG", "User Defined 8"}, + 0xF9: [2]string{"P16", "ProDOS-16 System File"}, + 0xFA: [2]string{"INT", "Integer BASIC Program"}, + 0xFB: [2]string{"IVR", "Integer BASIC Variables"}, + 0xFC: [2]string{"BAS", "Applesoft BASIC Program"}, + 0xFD: [2]string{"VAR", "Applesoft BASIC Variables"}, + 0xFE: [2]string{"REL", "EDASM Relocatable Code"}, + 0xFF: [2]string{"SYS", "ProDOS-8 System File"}, +} + +func (t ProDOSFileType) String() string { + + info, ok := ProDOSTypeMap[t] + if ok { + return info[1] + } + + return "Unknown" +} + +func (ft ProDOSFileType) Ext() string { + + info, ok := ProDOSTypeMap[ft] + if ok { + return info[0] + } + + return "BIN" +} + +func ProDOSFileTypeFromExt(ext string) ProDOSFileType { + for ft, info := range ProDOSTypeMap { + if strings.ToUpper(ext) == info[0] { + return ft + } + } + return 0x06 +} + +func (t ProDOSFileType) Valid() bool { + + if t == FileType_PD_None { + return false + } + + _, ok := ProDOSTypeMap[t] + + return ok +} + +func (fd *ProDOSFileDescriptor) Type() ProDOSFileType { + return ProDOSFileType(fd.Data[16]) +} + +func (fd *ProDOSFileDescriptor) SetType(t ProDOSFileType) { + fd.Data[16] = byte(t) +} + +func (fd *ProDOSFileDescriptor) IndexBlock() int { + return int(fd.Data[17]) + 256*int(fd.Data[18]) +} + +func (fd *ProDOSFileDescriptor) SetIndexBlock(b int) { + fd.Data[17] = byte(b & 0xff) + fd.Data[18] = byte(b / 256) +} + +func (fd *ProDOSFileDescriptor) TotalBlocks() int { + return int(fd.Data[19]) + 256*int(fd.Data[20]) +} + +func (fd *ProDOSFileDescriptor) SetTotalBlocks(b int) { + fd.Data[19] = byte(b & 0xff) + fd.Data[20] = byte(b / 256) +} + +func (fd *ProDOSFileDescriptor) AuxType() int { + return int(fd.Data[31]) + 256*int(fd.Data[32]) +} + +func (fd *ProDOSFileDescriptor) SetAuxType(b int) { + fd.Data[31] = byte(b & 0xff) + fd.Data[32] = byte(b / 256) +} + +func (fd *ProDOSFileDescriptor) TotalSectors() int { + return fd.TotalBlocks() / 2 +} + +func (fd *ProDOSFileDescriptor) Size() int { + return int(fd.Data[21]) + 256*int(fd.Data[22]) + 65536*int(fd.Data[23]) +} + +func (fd *ProDOSFileDescriptor) SetSize(v int) { + fd.Data[21] = byte(v & 0xff) + fd.Data[22] = byte((v >> 8) & 0xff) + fd.Data[23] = byte((v >> 16) & 0xff) +} + +func (fd *ProDOSFileDescriptor) SetHeaderPointer(v int) { + fd.Data[0x25] = byte(v & 0xff) + fd.Data[0x26] = byte((v >> 8) & 0xff) +} + +func (fd *ProDOSFileDescriptor) HeaderPointer() int { + return int(fd.Data[0x25]) + 256*int(fd.Data[0x26]) +} + +func (d *DSKWrapper) PRODOS800GetBlock(block int) ([]byte, error) { + + t, s1, s2 := d.PRODOS800GetBlockSectors(block) + + e := d.Seek(t, s1) + if e != nil { + return []byte(nil), e + } + data := make([]byte, 0) + c1 := d.Read() + data = append(data, c1...) + + e = d.Seek(t, s2) + if e != nil { + return []byte(nil), e + } + c2 := d.Read() + data = append(data, c2...) + + return data, nil +} + +func (d *DSKWrapper) PRODOSGetBlock(block int) ([]byte, error) { + + t, s1, s2 := d.PRODOSGetBlockSectors(block) + + e := d.Seek(t, s1) + if e != nil { + return []byte(nil), e + } + data := make([]byte, 0) + c1 := d.Read() + data = append(data, c1...) + + e = d.Seek(t, s2) + if e != nil { + return []byte(nil), e + } + c2 := d.Read() + data = append(data, c2...) + + return data, nil +} + +func (d *DSKWrapper) PRODOS800GetVDH(b int) (*VDH, error) { + + data, e := d.PRODOS800GetBlock(b) + if e != nil { + return nil, e + } + + vdh := &VDH{} + vdh.SetData(data[4:43], b, 4) + return vdh, nil + +} + +func (d *DSKWrapper) PRODOSGetVDH(b int) (*VDH, error) { + + data, e := d.PRODOSGetBlock(b) + if e != nil { + return nil, e + } + + vdh := &VDH{} + vdh.SetData(data[4:43], b, 4) + return vdh, nil + +} + +func (d *DSKWrapper) PRODOSGetCatalogPathed(start int, path string, pattern string) (*VDH, []ProDOSFileDescriptor, error) { + + path = strings.Trim(path, "/") + //fmt.Printf("PRODOSGetCatalogPathed(%d, \"%s\", \"%s\" )\n", start, path, pattern) + + //start := 2 // where we start our descent + + if path != "" { + parts := strings.Split(strings.Trim(path, "/"), "/") + + subdir := parts[0] + parts = parts[1:] + + // find subdirectories + vdh, files, e := d.PRODOSGetCatalog(start, subdir) + if e != nil { + return vdh, files, e + } + if len(files) != 1 { + return vdh, files, e + } + if files[0].Type() != FileType_PD_Directory { + return vdh, files, errors.New("Not a directory") + } + + // ok, we found the directory... + if len(parts) > 0 { + // more subdirs + //fmt.Printf(">> Entering prodos subdir [%s]\n", files[0].Name()) + newpathstr := strings.Join(parts, "/") + return d.PRODOSGetCatalogPathed(files[0].IndexBlock(), newpathstr, pattern) + } else { + // get directory based on this block + //fmt.Printf("-- Entering prodos subdir [%s]\n", files[0].Name()) + return d.PRODOSGetCatalog(files[0].IndexBlock(), pattern) + } + + } else { + return d.PRODOSGetCatalog(start, pattern) + } + +} + +func (d *DSKWrapper) PRODOSGetCatalog(startblock int, pattern string) (*VDH, []ProDOSFileDescriptor, error) { + + //fmt.Printf("GetCatalogProDOS(%d, %s)\n", startblock, pattern) + + var err error + var re *regexp.Regexp + var patterntmp string + if pattern != "" { + patterntmp = strings.Replace(pattern, ".", "[.]", -1) + patterntmp = strings.Replace(patterntmp, "*", ".*", -1) + patterntmp = "(?i)^" + patterntmp + "$" + re = regexp.MustCompile(patterntmp) + } + + var files []ProDOSFileDescriptor + var e error + var vtoc *VDH + + if d.Format.ID == DF_PRODOS_800KB { + vtoc, e = d.PRODOS800GetVDH(startblock) + } else { + vtoc, e = d.PRODOSGetVDH(startblock) + } + if e != nil { + return vtoc, files, e + } + + activeentries := 0 + filecount := vtoc.GetFileCount() + blockentries := 2 + entriesperblock := vtoc.GetEntriesPerBlock() + refnum := startblock + + var data []byte + if d.Format.ID == DF_PRODOS_800KB { + data, _ = d.PRODOS800GetBlock(refnum) + } else { + data, _ = d.PRODOSGetBlock(refnum) + } + + nextblock := int(data[2]) + 256*int(data[3]) + + //fmt.Printf("ActiveCount = %d\n", filecount) + + entrypointer := 4 + PRODOS_ENTRY_SIZE + + for activeentries < filecount { + + if data[entrypointer] != 0x00 { + // Valid entry + chunk := data[entrypointer : entrypointer+PRODOS_ENTRY_SIZE] + fd := ProDOSFileDescriptor{} + fd.SetData(chunk, refnum, entrypointer) + + if fd.Type().Valid() { + + var skipname bool = false + if re != nil { + //fmt.Printf("Checking [%s] against regex /%s/\n", fd.Name(), patterntmp) + skipname = !re.MatchString(fd.Name()) + } + + if fd.GetStorageType() != StorageType_Inactive && !skipname { + files = append(files, fd) + } + + } + + activeentries++ + } + + if activeentries < filecount { + if blockentries == entriesperblock { + refnum = nextblock + if d.Format.ID == DF_PRODOS_800KB { + data, err = d.PRODOS800GetBlock(refnum) + } else { + data, err = d.PRODOSGetBlock(refnum) + } + if err != nil { + break + } + nextblock = int(data[2]) + 256*int(data[3]) + blockentries = 0x01 + entrypointer = 0x04 + } else { + entrypointer += PRODOS_ENTRY_SIZE + blockentries++ + } + } + + } + + return vtoc, files, nil +} + +func (d *DSKWrapper) PRODOSReadFileSectors(fd ProDOSFileDescriptor, maxblocks int) ([]byte, error) { + + var data, index, chunk []byte + var e error + + switch fd.GetStorageType() { + case StorageType_Seedling: + /* single block pointed to */ + if d.Format.ID == DF_PRODOS_800KB { + data, _ = d.PRODOS800GetBlock(fd.IndexBlock()) + } else { + data, _ = d.PRODOSGetBlock(fd.IndexBlock()) + } + count := fd.Size() + if count > len(data) { + count = len(data) + } + return data[:count], e + case StorageType_Sapling: + if d.Format.ID == DF_PRODOS_800KB { + index, _ = d.PRODOS800GetBlock(fd.IndexBlock()) + } else { + index, _ = d.PRODOSGetBlock(fd.IndexBlock()) + } + data := make([]byte, 0) + bptr := 0 + remaining := fd.Size() - len(data) + for len(data) < fd.Size() && bptr+256 < len(index) { + blocknum := int(index[bptr]) + 256*int(index[bptr+256]) + + //fmt.Printf("File block %d (%d %d)\n", blocknum, int(index[bptr]), int(index[bptr+1])) + + if d.Format.ID == DF_PRODOS_800KB { + chunk, e = d.PRODOS800GetBlock(blocknum) + } else { + chunk, e = d.PRODOSGetBlock(blocknum) + } + if e != nil { + return data, e + } + + count := 512 + if remaining < count { + count = remaining + } + + data = append(data, chunk[:count]...) + + bptr += 1 + remaining = fd.Size() - len(data) + } + return data, e + case StorageType_Tree: + return []byte(nil), errors.New("Unimplemented yet... soz :(") + } + + return []byte(nil), nil +} + +func (d *DSKWrapper) PRODOSReadFile(fd ProDOSFileDescriptor) (int, int, []byte, error) { + + data, e := d.PRODOSReadFileSectors(fd, -1) + + if e != nil { + return 0, 0, data, e + } + + switch fd.Type() { + case FileType_PD_INT: + return fd.Size(), fd.AuxType(), IntegerDetoks(data), nil + case FileType_PD_APP: + return fd.Size(), fd.AuxType(), ApplesoftDetoks(data), nil + case FileType_PD_TXT: + return fd.Size(), fd.AuxType(), data, nil + case FileType_PD_BIN: + return fd.Size(), fd.AuxType(), data, nil + default: + return fd.Size(), fd.AuxType(), data, nil + } + +} + +func (d *DSKWrapper) PRODOSReadFileRaw(fd ProDOSFileDescriptor) (int, int, []byte, error) { + + data, e := d.PRODOSReadFileSectors(fd, -1) + + if e != nil { + return 0, 0, data, e + } + + switch fd.Type() { + case FileType_PD_INT: + return fd.Size(), fd.AuxType(), data, nil + case FileType_PD_APP: + return fd.Size(), fd.AuxType(), data, nil + case FileType_PD_TXT: + return fd.Size(), fd.AuxType(), data, nil + case FileType_PD_BIN: + return fd.Size(), fd.AuxType(), data, nil + default: + return fd.Size(), fd.AuxType(), data, nil + } + +} + +func (d *DSKWrapper) PRODOSChecksumBlock(b int) string { + bl, _ := d.PRODOSGetBlock(b) + return Checksum(bl) +} + +func (d *DSKWrapper) PRODOS800ChecksumBlock(b int) string { + bl, _ := d.PRODOS800GetBlock(b) + return Checksum(bl) +} + +func (d *DSKWrapper) PRODOSGetBlockSectors(block int) (int, int, int) { + + track := block / PRODOS_BLOCKS_PER_TRACK + + bo := block % PRODOS_BLOCKS_PER_TRACK + + if d.Layout == SectorOrderProDOSLinear { + return track, bo * 2, bo*2 + 1 + } + + switch bo { + case 0: + return track, 0x0, 0xe + case 1: + return track, 0xd, 0xc + case 2: + return track, 0xb, 0xa + case 3: + return track, 0x9, 0x8 + case 4: + return track, 0x7, 0x6 + case 5: + return track, 0x5, 0x4 + case 6: + return track, 0x3, 0x2 + case 7: + return track, 0x1, 0xf + } + + return track, 0x0, 0xe + +} + +func (d *DSKWrapper) PRODOS800GetBlockSectors(block int) (int, int, int) { + + dblBlock := block * 2 + + spt := 40 + + track := dblBlock / spt + s1 := dblBlock % spt + s2 := (dblBlock + 1) % spt + + return track, s1, s2 + +} + +// PRODOSGetDirBlocks returns a list of blocks for the current directory level +func (dsk *DSKWrapper) PRODOSGetDirBlocks(start int) (*VDH, []int, [][]byte, error) { + blocks := make([]int, 0) + chunks := make([][]byte, 0) + + vdh, err := dsk.PRODOSGetVDH(start) + if err != nil { + return vdh, blocks, chunks, err + } + + // okay lets trail the directory + blocks = append(blocks, start) + data, _ := dsk.PRODOSGetBlock(start) + chunks = append(chunks, data) + nextBlock := int(data[0x02]) + 256*int(data[0x03]) + for nextBlock != 0 { + blocks = append(blocks, nextBlock) + data, _ := dsk.PRODOSGetBlock(nextBlock) + chunks = append(chunks, data) + nextBlock = int(data[0x02]) + 256*int(data[0x03]) + } + + return vdh, blocks, chunks, nil + +} + +// PRODOSFindDirBlocks locates the blocks for a given directory (if it exists!) +func (dsk *DSKWrapper) PRODOSFindDirBlocks(start int, path string) (*VDH, []int, [][]byte, error) { + + path = strings.Trim(path, "/") + + if path == "" { + return dsk.PRODOSGetDirBlocks(start) + } + + segments := strings.Split(path, "/") + target := segments[0] + segments = segments[1:] + + vdh, files, err := dsk.PRODOSGetCatalog(start, "") + if err != nil { + return vdh, nil, nil, err + } + + for _, f := range files { + + if f.Type() != FileType_PD_Directory { + continue + } + + if strings.ToLower(target) == strings.ToLower(f.NameUnadorned()) { + // matched path + newpath := strings.Join(segments, "/") + return dsk.PRODOSFindDirBlocks(f.IndexBlock(), newpath) + } + + } + + return vdh, nil, nil, errors.New("Path not found") + +} + +// PRODOSGetFirstFreeEntry for a given path, will find the first free file descriptor +// it will expand the directory if needed to create a free block +func (dsk *DSKWrapper) PRODOSGetFirstFreeEntry(path string, name string, grow bool) (*ProDOSFileDescriptor, error) { + + vdh, blockList, blockData, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return nil, err + } + + entries := 0 + + // if we are here, we found the right directory... + for idx, data := range blockData { + count := 0 + + if idx == 0 { + count = 1 + entries = 1 + } + + for count < vdh.GetEntriesPerBlock() { + offset := 4 + count*PRODOS_ENTRY_SIZE + chunk := data[offset : offset+PRODOS_ENTRY_SIZE] + fd := &ProDOSFileDescriptor{} + fd.SetData(chunk, blockList[idx], offset) + fd.entryNum = entries + + //Printf("--> Check entry: %s, %v, %v\n", fd.NameUnadorned(), fd.CreateTime(), fd.ModTime()) + + if fd.GetStorageType() != 0x00 && strings.ToLower(fd.NameUnadorned()) == strings.ToLower(name) { + return fd, nil + } else if fd.GetStorageType() == 0x00 { + //fmt.Printf("found at entry %d in block %d\n", entries, blockList[idx]) + return fd, nil + } + + count += 1 + entries++ + } + } + + if !grow { + return nil, errors.New("No free slot: told not to grow directory") + } + + // If we got here, we need to create a new block + freeBlocks, err := dsk.PRODOSGetFreeBlocks(1, vdh.GetTotalBlocks()) + if err != nil { + return nil, errors.New("Could not extend directory") + } + + data, err := dsk.PRODOSGetBlock(freeBlocks[0]) + prevBlock := blockList[len(blockList)-1] + data[0x00] = byte(prevBlock & 0xff) + data[0x01] = byte(prevBlock / 0x100) + + offset := 4 + chunk := data[offset : offset+PRODOS_ENTRY_SIZE] + fd := &ProDOSFileDescriptor{} + fd.entryNum = entries + fd.SetData(chunk, freeBlocks[0], offset) + + dsk.PRODOSMarkBlocks(freeBlocks, false) + + return fd, nil +} + +func (dsk *DSKWrapper) PRODOSMarkBlocks(list []int, free bool) error { + vbm, err := dsk.PRODOSGetVolumeBitmap() + if err != nil { + return err + } + + for _, b := range list { + vbm.SetBlockFree(b, free) + } + + //fmt.Printf("Writing Volume bitmap to block %d\n", vbm.blockid) + + return dsk.PRODOSWrite(vbm.blockid, vbm.Data) +} + +func (dsk *DSKWrapper) PRODOSGetFreeBlocks(count int, totalBlocks int) ([]int, error) { + + vbm, err := dsk.PRODOSGetVolumeBitmap() + if err != nil { + return nil, err + } + b := 0 + + blocks := make([]int, 0) + + for b < totalBlocks && len(blocks) < count { + if vbm.IsBlockFree(b) { + blocks = append(blocks, b) + } + b++ + } + + if len(blocks) == count { + return blocks, nil + } + + return blocks, errors.New("Not enough blocks") + +} + +func (dsk *DSKWrapper) PRODOSDeleteFile(path string, name string) error { + + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err != nil { + return err + } + + if fd.GetStorageType() == 0x00 { + return errors.New("Not found") + } + + // At this stage we have a match + access := fd.AccessMode() + if access&AccessType_Destroy == 0 { + return errors.New("Permission denied") + } + if access&AccessType_Writable == 0 { + return errors.New("Read-only file") + } + + // Make sure its either a sapling or a seedling + st := fd.GetStorageType() + if st != StorageType_Sapling && st != StorageType_Seedling && st != StorageType_SubDir_File { + return errors.New("Special file deletion not implemented: yet.") + } else if st == StorageType_SubDir_File { + return dsk.PRODOSDeleteDirectory(path, name) + } + + var removeBlocks []int + switch st { + case StorageType_Seedling: + removeBlocks = append(removeBlocks, fd.IndexBlock()) + case StorageType_Sapling: + removeBlocks = append(removeBlocks, fd.IndexBlock()) + ib, err := dsk.PRODOSGetBlock(removeBlocks[0]) + if err != nil { + return err + } + + i := 0 + b := int(ib[2*i+0]) + 256*int(ib[2*i+1]) + for i < 256 && b != 0 { + b = int(ib[2*i+0]) + 256*int(ib[2*i+1]) + i++ + } + } + + err = dsk.PRODOSMarkBlocks(removeBlocks, true) + if err != nil { + return err + } + + // // get the VDH -- we need this later + vdh, _, _, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return err + } + + // now delete the fileentry + fd.SetStorageType(StorageType_Inactive) + err = fd.Publish(dsk) + if err != nil { + return err + } + + // update filecount + vdh.SetFileCount(vdh.GetFileCount() - 1) + + return vdh.Publish(dsk) + +} + +func (dsk *DSKWrapper) PRODOSWriteFile(path string, name string, kind ProDOSFileType, data []byte, auxtype int) error { + + name = strings.ToUpper(name) + + nst := StorageType_Seedling + blocksNeeded := len(data)/512 + 1 + totalBlocks := blocksNeeded + if blocksNeeded > 1 { + nst = StorageType_Sapling + totalBlocks++ // extra block for blocklist + } else if blocksNeeded > 256 { + return errors.New("Not implemented: Tree write") + } + + var origTime time.Time + var origAccess ProDOSAccessMode + + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err == nil { + origTime = fd.CreateTime() // we need this later + origAccess = fd.AccessMode() + err = dsk.PRODOSDeleteFile(path, name) + if err != nil { + return err + } + } else { + + fd, err = dsk.PRODOSGetFirstFreeEntry(path, name, true) + if err != nil { + return err + } + + } + + vdh, err := dsk.PRODOSGetVDH(2) + if err != nil { + return err + } + freeBlocks, err := dsk.PRODOSGetFreeBlocks(totalBlocks, vdh.GetTotalBlocks()) + if err != nil { + return err + } + + // Okay got enough blocks + switch nst { + case StorageType_Sapling: + err = dsk.PRODOSWriteSaplingBlocks(freeBlocks[0], freeBlocks[1:], data) + if err != nil { + return err + } + case StorageType_Seedling: + //fmt.Printf("Write Seedling %d bytes data to block %d\n", len(data), freeBlocks[0]) + err = dsk.PRODOSWrite(freeBlocks[0], data) + if err != nil { + return err + } + } + + // Get the current directories directory header + dvdh, blocks, _, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return err + } + + // common details - note we preserve access mode and time when overwriting the same file + fd.SetAuxType(auxtype) + fd.SetName(name) + fd.SetType(kind) + fd.SetTotalBlocks(totalBlocks) + fd.SetIndexBlock(freeBlocks[0]) + fd.SetSize(len(data)) + fd.SetStorageType(nst) + if origAccess == 0x00 { + fd.SetAccessMode(AccessType_Default) + } else { + fd.SetAccessMode(origAccess) + } + if origTime.IsZero() { + fd.SetCreateTime(time.Now()) + } else { + fd.SetCreateTime(origTime) + } + fd.SetModTime(time.Now()) + fd.SetHeaderPointer(blocks[0]) + + fd.Publish(dsk) + + dvdh.SetFileCount(vdh.GetFileCount() + 1) + dvdh.Publish(dsk) + + err = dsk.PRODOSMarkBlocks(freeBlocks, false) + if err != nil { + return err + } + + return nil +} + +func (fd *ProDOSFileDescriptor) Publish(dsk *DSKWrapper) error { + + //fmt.Printf("Writing FD data back to block %d, offset %d\n", fd.blockid, fd.blockoffset) + + bd, err := dsk.PRODOSGetBlock(fd.blockid) + if err != nil { + return err + } + for i, v := range fd.Data { + bd[fd.blockoffset+i] = v + } + // for i, _ := range bd { + // bd[i] = byte(0xff ^ (i % 2)) + // } + + //Dump(bd) + + return dsk.PRODOSWrite(fd.blockid, bd) +} + +func (dsk *DSKWrapper) PRODOSWrite(b int, data []byte) error { + + for len(data) < 512 { + data = append(data, 0x00) + } + + t, s1, s2 := dsk.PRODOSGetBlockSectors(b) + + err := dsk.Seek(t, s1) + if err != nil { + return err + } + dsk.Write(data[:256]) + err = dsk.Seek(t, s2) + if err != nil { + return err + } + dsk.Write(data[256:]) + + return nil + +} + +func (dsk *DSKWrapper) PRODOSWriteSaplingBlocks(indexBlock int, dataBlocks []int, data []byte) error { + + if len(dataBlocks) > 256 || len(data) > 0x20000 { + return errors.New("Too many data blocks") + } + + ib := make([]byte, 512) + for i, blocknum := range dataBlocks { + // index the block + ib[i*2+0] = byte(blocknum & 0xff) + ib[i*2+1] = byte(blocknum / 0x100) + + // data offset... + ptr := 512 * i + end := ptr + 512 + if end > len(data) { + end = len(data) + } + chunk := data[ptr:end] + for len(chunk) < 512 { + chunk = append(chunk, 0x00) + } + err := dsk.PRODOSWrite(blocknum, chunk) + if err != nil { + return err + } + } + + return dsk.PRODOSWrite(indexBlock, ib) +} + +// PRODOSCreateDirectory tries to create a subdirectory... +func (dsk *DSKWrapper) PRODOSCreateDirectory(path string, name string) error { + + vdh, err := dsk.PRODOSGetVDH(2) + if err != nil { + return err + } + + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err == nil { + return errors.New("Item exists") + } + + fd, err = dsk.PRODOSGetFirstFreeEntry(path, name, true) + if err != nil { + return err + } + + // got file descriptor + if fd.GetStorageType() != 0 { + + if fd.Type() == FileType_PD_Directory { + return errors.New("Directory already exists") + } + + return errors.New("Type mismatch") + } + + // Get the current directories directory header + dvdh, blocks, _, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return err + } + + // find a freeBlock for the directory + freeBlocks, err := dsk.PRODOSGetFreeBlocks(1, vdh.GetTotalBlocks()) + if err != nil { + return err + } + + // got a file descriptor + fd.SetStorageType(StorageType_SubDir_File) + fd.SetAccessMode(AccessType_Default | AccessType_Changed) + fd.SetCreateTime(time.Now()) + fd.SetModTime(time.Now()) + fd.SetName(name) + fd.SetType(FileType_PD_Directory) + fd.SetIndexBlock(freeBlocks[0]) // <------- block for the directory header + fd.SetTotalBlocks(1) + fd.SetSize(512) + fd.SetHeaderPointer(blocks[0]) + fd.SetMinVersion(0x00) + fd.SetVersion(0x23) + + err = dsk.PRODOSInitDirectoryBlock(freeBlocks[0], blocks[0], dvdh.GetEntriesPerBlock(), dvdh.GetEntryLength(), fd.entryNum, name) + if err != nil { + return err + } + + err = dsk.PRODOSMarkBlocks(freeBlocks, false) + if err != nil { + return err + } + + dvdh.SetFileCount(vdh.GetFileCount() + 1) + dvdh.Publish(dsk) + if err != nil { + return err + } + + return fd.Publish(dsk) + +} + +func (dsk *DSKWrapper) PRODOSInitDirectoryBlock(targetBlock int, parentBlock int, entriesPerBlock int, entrySize int, entryNum int, name string) error { + + block := make([]byte, 512) + dh := &VDH{ + Data: block[4 : 4+PRODOS_ENTRY_SIZE], + blockid: targetBlock, + blockoffset: 4, + } + + // Must be set (beneath Apple ProDOS) + dh.Data[0x10] = 0x75 + + dh.SetStorageType(StorageType_SubDir_Header) + dh.SetName(name) + dh.SetCreateTime(time.Now()) + dh.SetAccess(AccessType_Default) + dh.SetEntriesPerBlock(entriesPerBlock) + dh.SetEntryLength(entrySize) + dh.SetFileCount(0) + dh.SetDirParentPointer(parentBlock) + dh.SetDirParentEntry(entryNum) + dh.SetDirParentEntryLength(entrySize) + dh.SetMinVersion(0x00) + dh.SetVersion(0x23) + + return dsk.PRODOSWrite(targetBlock, block) + +} + +func (dsk *DSKWrapper) PRODOSDeleteDirectory(path string, name string) error { + + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err != nil { + return err + } + + // got file descriptor, is it empty + if fd.GetStorageType() == 0x00 { + return errors.New("Path not found") + } + + if fd.Type() != FileType_PD_Directory { + return errors.New("Not a directory") + } + + // Get the current directories directory header + _, files, err := dsk.PRODOSGetCatalog(fd.IndexBlock(), "*") + if err != nil { + return err + } + + // Remove any files in subdirectory + for _, subfile := range files { + if subfile.GetStorageType() == StorageType_SubDir_File { + // FOLDER, RECURSE + err = dsk.PRODOSDeleteDirectory(path+"/"+name, subfile.NameUnadorned()) + if err != nil { + return err + } + } else if subfile.GetStorageType() == StorageType_Tree { + // TREE FILE -- UNSUPPORTED FOR NOW + return errors.New("Tree file handling not supported currently") + } else { + err = dsk.PRODOSDeleteFile(path+"/"+name, subfile.NameUnadorned()) + if err != nil { + return err + } + } + } + + // Check again and make sure it is empty + _, files, err = dsk.PRODOSGetCatalog(fd.IndexBlock(), "*") + if err != nil { + return err + } + if len(files) > 0 { + return errors.New("Could not delete all subfiles") + } + + // Get blocks that make up the directory + _, dirBlocks, _, err := dsk.PRODOSFindDirBlocks(2, path+"/"+name) + if err != nil { + return err + } + + // Free all the things! + err = dsk.PRODOSMarkBlocks(dirBlocks, true) + if err != nil { + return err + } + + // free file entry + fd.SetStorageType(StorageType_Inactive) + err = fd.Publish(dsk) + if err != nil { + return err + } + + // reduce count by 1 at top level + tvdh, _, _, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return err + } + tvdh.SetFileCount(tvdh.GetFileCount() - 1) + return tvdh.Publish(dsk) + +} + +func (dsk *DSKWrapper) PRODOSSetLocked(path, name string, lock bool) error { + + // We cheat here a bit and use the get first free entry call with + // autogrow turned off. + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err != nil { + return err + } + + fd.SetLocked(lock) + return fd.Publish(dsk) + +} + +// PRODOSGetNamedEntry for a given path, will find the file descriptor with name +func (dsk *DSKWrapper) PRODOSGetNamedEntry(path string, name string) (*ProDOSFileDescriptor, error) { + + //fmt.Printf(">>> Path = %s\n", path) + + vdh, blockList, blockData, err := dsk.PRODOSFindDirBlocks(2, path) + if err != nil { + return nil, err + } + + entries := 0 + + // if we are here, we found the right directory... + for idx, data := range blockData { + count := 0 + + if idx == 0 { + count = 1 + entries = 1 + } + + for count < vdh.GetEntriesPerBlock() { + offset := 4 + count*PRODOS_ENTRY_SIZE + chunk := data[offset : offset+PRODOS_ENTRY_SIZE] + fd := &ProDOSFileDescriptor{} + fd.SetData(chunk, blockList[idx], offset) + fd.entryNum = entries + + //fmt.Printf(">>> Comparing %s to %s\n", strings.ToLower(fd.NameUnadorned()), strings.ToLower(name)) + + if fd.GetStorageType() != 0x00 && strings.ToLower(fd.NameUnadorned()) == strings.ToLower(name) { + return fd, nil + } + + count += 1 + entries++ + } + } + + return nil, errors.New("Not found") +} + +func (dsk *DSKWrapper) PRODOSRenameFile(path, name, newname string) error { + + fd, err := dsk.PRODOSGetNamedEntry(path, name) + if err != nil { + return err + } + + _, err = dsk.PRODOSGetNamedEntry(path, newname) + if err == nil { + return errors.New("New name already exists") + } + + // can rename here + fd.SetName(newname) + return fd.Publish(dsk) + +} diff --git a/disk/diskimagerdos.go b/disk/diskimagerdos.go new file mode 100644 index 0000000..7f0d47f --- /dev/null +++ b/disk/diskimagerdos.go @@ -0,0 +1,382 @@ +package disk + +import ( + "bytes" + "regexp" + "strings" +) + +const RDOS_CATALOG_TRACK = 0x01 +const RDOS_CATALOG_LENGTH = 0xB +const RDOS_ENTRY_LENGTH = 0x20 +const RDOS_NAME_LENGTH = 0x18 + +var RDOS_SIGNATURE = []byte{ + byte('R' + 0x80), + byte('D' + 0x80), + byte('O' + 0x80), + byte('S' + 0x80), + byte(' ' + 0x80), +} + +var RDOS_SIGNATURE_32 = []byte{ + byte('R' + 0x80), + byte('D' + 0x80), + byte('O' + 0x80), + byte('S' + 0x80), + byte(' ' + 0x80), + byte('2' + 0x80), +} + +var RDOS_SIGNATURE_33 = []byte{ + byte('R' + 0x80), + byte('D' + 0x80), + byte('O' + 0x80), + byte('S' + 0x80), + byte(' ' + 0x80), + byte('3' + 0x80), +} + +type RDOSFormatSpec struct { + SectorStride int + SectorMax int + CatalogTrack int + CatalogSector int + Ordering SectorOrder +} + +type RDOSFormat int + +const ( + RDOS_Unknown RDOSFormat = iota + RDOS_3 + RDOS_32 + RDOS_33 +) + +func (f RDOSFormat) String() string { + + switch f { + case RDOS_3: + return "RDOS3" + case RDOS_32: + return "RDOS32" + case RDOS_33: + return "RDOS33" + } + + return "Unknown" + +} + +func (f RDOSFormat) Spec() *RDOSFormatSpec { + + switch f { + case RDOS_32: + return &RDOSFormatSpec{ + SectorStride: 13, + SectorMax: 13, + CatalogTrack: 1, + CatalogSector: 0, + Ordering: SectorOrderDOS33, + } + case RDOS_3: + return &RDOSFormatSpec{ + SectorStride: 16, + SectorMax: 13, + CatalogTrack: 0, + CatalogSector: 1, + Ordering: SectorOrderDOS33, + } + case RDOS_33: + return &RDOSFormatSpec{ + SectorStride: 16, + SectorMax: 16, + CatalogTrack: 1, + CatalogSector: 12, + Ordering: SectorOrderProDOS, + } + } + return nil + +} + +func (dsk *DSKWrapper) IsRDOS() (bool, RDOSFormat) { + + // It needs to be either 140K or 113K + if len(dsk.Data) != STD_DISK_BYTES && len(dsk.Data) != STD_DISK_BYTES_OLD { + return false, RDOS_Unknown + } + + sectorStride := (len(dsk.Data) / STD_TRACKS_PER_DISK) / 256 + + idbytes := dsk.Data[sectorStride*256 : sectorStride*256+6] + + if bytes.Compare(idbytes, RDOS_SIGNATURE_32) == 0 && sectorStride == 13 { + return true, RDOS_32 + } + + if bytes.Compare(idbytes, RDOS_SIGNATURE_32) == 0 && sectorStride == 16 { + return true, RDOS_3 + } + + if bytes.Compare(idbytes, RDOS_SIGNATURE_33) == 0 && sectorStride == 16 { + return true, RDOS_33 + } + + return false, RDOS_Unknown + +} + +type RDOSFileDescriptor struct { + data [RDOS_ENTRY_LENGTH]byte +} + +func (fd *RDOSFileDescriptor) SetData(in []byte) { + for i, b := range in { + if i < RDOS_ENTRY_LENGTH { + fd.data[i] = b + } + } +} + +func (fd *RDOSFileDescriptor) IsDeleted() bool { + + return fd.data[24] == 0xa0 || fd.data[0] == 0x80 + +} + +func (fd *RDOSFileDescriptor) IsUnused() bool { + + return fd.data[24] == 0x00 + +} + +func (fd *RDOSFileDescriptor) IsLocked() bool { + return true +} + +type RDOSFileType int + +const ( + FileType_RDOS_Unknown RDOSFileType = iota + FileType_RDOS_AppleSoft + FileType_RDOS_Binary + FileType_RDOS_Text +) + +var RDOSTypeMap = map[RDOSFileType][2]string{ + FileType_RDOS_Unknown: [2]string{"UNK", "Unknown"}, + FileType_RDOS_AppleSoft: [2]string{"APP", "Applesoft Basic Program"}, + FileType_RDOS_Binary: [2]string{"BIN", "Binary File"}, + FileType_RDOS_Text: [2]string{"TXT", "ASCII Text"}, +} + +func (ft RDOSFileType) String() string { + info, ok := RDOSTypeMap[ft] + if ok { + return info[1] + } + return "Unknown" +} + +func (ft RDOSFileType) Ext() string { + info, ok := RDOSTypeMap[ft] + if ok { + return info[0] + } + return "UNK" +} + +func RDOSFileTypeFromExt(ext string) RDOSFileType { + for ft, info := range RDOSTypeMap { + if strings.ToUpper(ext) == info[0] { + return ft + } + } + return 0x00 +} + +func (fd *RDOSFileDescriptor) Type() RDOSFileType { + + switch rune(fd.data[24]) { + case 'A' + 0x80: + return FileType_RDOS_AppleSoft + case 'B' + 0x80: + return FileType_RDOS_Binary + case 'T' + 0x80: + return FileType_RDOS_Text + } + + return FileType_RDOS_Unknown + +} + +func (fd *RDOSFileDescriptor) Name() string { + + str := "" + for i := 0; i < RDOS_NAME_LENGTH; i++ { + + ch := rune(fd.data[i] & 127) + if ch == 0 { + break + } + str += string(ch) + + } + + str = strings.TrimRight(str, " ") + switch fd.Type() { + case FileType_RDOS_AppleSoft: + str += ".a" + case FileType_RDOS_Binary: + str += ".s" + case FileType_RDOS_Text: + str += ".t" + } + + return str + +} + +func (fd *RDOSFileDescriptor) NameUnadorned() string { + + str := "" + for i := 0; i < RDOS_NAME_LENGTH; i++ { + + ch := rune(fd.data[i] & 127) + if ch == 0 { + break + } + str += string(ch) + + } + + return str + +} + +func (fd RDOSFileDescriptor) NumSectors() int { + return int(fd.data[25]) +} + +func (fd RDOSFileDescriptor) LoadAddress() int { + return int(fd.data[26]) + 256*int(fd.data[27]) +} + +func (fd RDOSFileDescriptor) StartSector() int { + return int(fd.data[30]) + 256*int(fd.data[31]) +} + +func (fd RDOSFileDescriptor) Length() int { + return int(fd.data[28]) + 256*int(fd.data[29]) +} + +func (dsk *DSKWrapper) RDOSGetCatalog(pattern string) ([]*RDOSFileDescriptor, error) { + + pattern = strings.Replace(pattern, ".", "[.]", -1) + pattern = strings.Replace(pattern, "*", ".*", -1) + pattern = strings.Replace(pattern, "?", ".", -1) + + rx := regexp.MustCompile("(?i)" + pattern) + + var files = make([]*RDOSFileDescriptor, 0) + + d := make([]byte, 0) + + for s := 0; s < RDOS_CATALOG_LENGTH; s++ { + dsk.SetTrack(1) + dsk.SetSector(s) + chunk := dsk.Read() + d = append(d, chunk...) + } + + var dirPtr int + for i := 0; i < RDOS_CATALOG_LENGTH*RDOS_ENTRY_LENGTH; i++ { + entry := &RDOSFileDescriptor{} + entry.SetData(d[dirPtr : dirPtr+RDOS_ENTRY_LENGTH]) + + dirPtr += RDOS_ENTRY_LENGTH + + if entry.IsUnused() { + break + } + + if !entry.IsDeleted() && rx.MatchString(entry.NameUnadorned()) { + files = append(files, entry) + } + + } + + return files, nil + +} + +func (dsk *DSKWrapper) RDOSUsedBitmap() ([]bool, error) { + + spt := dsk.RDOSFormat.Spec().SectorMax + activeSectors := spt * 35 + + used := make([]bool, activeSectors) + + files, err := dsk.RDOSGetCatalog("*") + if err != nil { + return used, err + } + + for _, file := range files { + + length := file.NumSectors() + start := file.StartSector() + if start+length > activeSectors { + continue // file is bad + } + + for block := start; block < start+length; block++ { + used[block] = true + } + + } + + return used, nil + +} + +func (dsk *DSKWrapper) RDOSReadFile(file *RDOSFileDescriptor) ([]byte, error) { + + spt := dsk.RDOSFormat.Spec().SectorMax + activeSectors := spt * 35 + + length := file.NumSectors() + start := file.StartSector() + + // If file is damaged return nothing + if start+length > activeSectors { + return []byte(nil), nil + } + + block := start + data := make([]byte, 0) + for block < start+length && len(data) < file.Length() { + + track := block / spt + sector := block % spt + + dsk.SetTrack(track) + dsk.SetSector(sector) + + chunk := dsk.Read() + needed := file.Length() - len(data) + if needed >= 256 { + data = append(data, chunk...) + } else { + data = append(data, chunk[:needed]...) + } + + block++ + + } + + return data, nil + +} diff --git a/disk/int.go b/disk/int.go new file mode 100644 index 0000000..9e18a06 --- /dev/null +++ b/disk/int.go @@ -0,0 +1,33 @@ +package disk + +import "time" + +type CatalogEntryType int + +const ( + CETUnknown CatalogEntryType = iota + CETBinary + CETBasicApplesoft + CETBasicInteger + CETPascal + CETText + CETData + CETGraphics +) + +type CatalogEntry interface { + Size() int // file size in bytes + Name() string + NameUnadorned() string + Date() time.Time + Type() CatalogEntryType +} + +type DiskImage interface { + IsValid() (bool, DiskFormat, SectorOrder) + GetCatalog(path string, pattern string) ([]CatalogEntry, error) + ReadFile(fd CatalogEntry) (int, []byte, error) + StoreFile(fd CatalogEntry) error + GetUsedBitmap() ([]bool, error) + Nibblize() ([]byte, error) +} diff --git a/drvappledos13.go b/drvappledos13.go new file mode 100644 index 0000000..641cf20 --- /dev/null +++ b/drvappledos13.go @@ -0,0 +1,151 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "time" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzeDOS13(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + l.Logf("Reading Disk VTOC...") + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + l.Errorf("Error reading VTOC: %s", err.Error()) + return + } + + info.Tracks, info.Sectors = vtoc.GetTracks(), vtoc.GetSectors() + l.Logf("Tracks: %d, Sectors: %d", info.Tracks, info.Sectors) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Tracks*info.Sectors) + + info.ActiveSectors = make(DiskSectors, 0) + info.InactiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + for t := 0; t < info.Tracks; t++ { + + for s := 0; s < info.Sectors; s++ { + info.Bitmap[t*info.Sectors+s] = !vtoc.IsTSFree(t, s) + + // checksum sector + //info.SectorFingerprints[dsk.ChecksumSector(t, s)] = &DiskBlock{Track: t, Sector: s} + + if info.Bitmap[t*info.Sectors+s] { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + activeData = append(activeData, data...) + + if *ingestMode&2 == 2 { + sector.Data = data + } + + info.ActiveSectors = append(info.ActiveSectors, sector) + } else { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + if *ingestMode&2 == 2 { + sector.Data = data + } + //activeData = append(activeData, data...) + + info.InactiveSectors = append(info.InactiveSectors, sector) + } + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // Analyzing files + l.Log("Starting Analysis of files") + + vtoc, files, err := dsk.AppleDOSGetCatalog("*") + if err != nil { + l.Errorf("Problem reading directory: %s", err.Error()) + return + } + + info.Files = make([]*DiskFile, 0) + for _, fd := range files { + l.Logf("- Name=%s, Type=%s", fd.NameUnadorned(), fd.Type()) + + file := DiskFile{ + Filename: fd.NameUnadorned(), + Type: fd.Type().String(), + Locked: fd.IsLocked(), + Ext: fd.Type().Ext(), + Created: time.Now(), + Modified: time.Now(), + } + + _, _, data, err := dsk.AppleDOSReadFileRaw(fd) + if err == nil { + sum := sha256.Sum256(data) + file.SHA256 = hex.EncodeToString(sum[:]) + file.Size = len(data) + if *ingestMode&1 == 1 { + if fd.Type() == disk.FileTypeAPP { + file.Text = disk.ApplesoftDetoks(data) + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.Data = data + file.LoadAddress = 0x801 + } else if fd.Type() == disk.FileTypeINT { + file.Text = disk.IntegerDetoks(data) + file.Data = data + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.LoadAddress = 0x1000 + } else if fd.Type() == disk.FileTypeTXT { + file.Text = disk.StripText(data) + file.Data = data + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.LoadAddress = 0x0000 + } else if fd.Type() == disk.FileTypeBIN && len(data) >= 2 { + file.LoadAddress = int(data[0]) + 256*int(data[1]) + file.Data = data[2:] + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + } else { + file.LoadAddress = 0x0000 + file.Data = data + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + } + } + } + + info.Files = append(info.Files, &file) + + } + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) +} diff --git a/drvappledos16.go b/drvappledos16.go new file mode 100644 index 0000000..a1f0abb --- /dev/null +++ b/drvappledos16.go @@ -0,0 +1,186 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "time" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzeDOS16(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + l.Logf("Reading Disk VTOC...") + vtoc, err := dsk.AppleDOSGetVTOC() + if err != nil { + l.Errorf("Error reading VTOC: %s", err.Error()) + return + } + + info.Tracks, info.Sectors = vtoc.GetTracks(), vtoc.GetSectors() + + if vtoc.BytesPerSector() != 256 { + l.Errorf("Disk does not seem to be AppleDOS - treat as generic") + dsk.Format = disk.GetDiskFormat(disk.DF_NONE) + analyzeNONE(id, dsk, info) + return + } + + l.Logf("Tracks: %d, Sectors: %d", info.Tracks, info.Sectors) + l.Logf("Sector order: %d", vtoc.GetTrackOrder()) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Tracks*info.Sectors) + + var useAlt bool + + if vtoc.IsTSFree(17, 0) { + info.Bitmap, _ = dsk.AppleDOSUsedBitmap() + useAlt = true + } + + info.ActiveSectors = make(DiskSectors, 0) + info.InactiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + for t := 0; t < info.Tracks; t++ { + + for s := 0; s < info.Sectors; s++ { + + if !useAlt { + info.Bitmap[t*info.Sectors+s] = !vtoc.IsTSFree(t, s) + } + + // checksum sector + //info.SectorFingerprints[dsk.ChecksumSector(t, s)] = &DiskBlock{Track: t, Sector: s} + + if info.Bitmap[t*info.Sectors+s] { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + activeData = append(activeData, data...) + + if *ingestMode&2 == 2 { + sector.Data = data + } + + info.ActiveSectors = append(info.ActiveSectors, sector) + } else { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + if *ingestMode&2 == 2 { + sector.Data = data + } + //activeData = append(activeData, data...) + + info.InactiveSectors = append(info.InactiveSectors, sector) + } + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // Analyzing files + l.Log("Starting Analysis of files") + + // lines := []string{ + // "10 PRINT \"HELLO WORLDS\"", + // "20 GOTO 10", + // } + + // e := dsk.AppleDOSWriteFile("CHEESE", disk.FileTypeAPP, disk.ApplesoftTokenize(lines), 0x801) + // if e != nil { + // l.Errorf("Error writing file: %s", e.Error()) + // panic(e) + // } + // f, _ := os.Create("out.dsk") + // f.Write(dsk.Data) + // f.Close() + + vtoc, files, err := dsk.AppleDOSGetCatalog("*") + if err != nil { + l.Errorf("Problem reading directory: %s", err.Error()) + return + } + + info.Files = make([]*DiskFile, 0) + for _, fd := range files { + l.Logf("- Name=%s, Type=%s", fd.NameUnadorned(), fd.Type()) + + file := DiskFile{ + Filename: fd.NameUnadorned(), + Type: fd.Type().String(), + Locked: fd.IsLocked(), + Ext: fd.Type().Ext(), + Created: time.Now(), + Modified: time.Now(), + } + + //l.Log("start read") + _, _, data, err := dsk.AppleDOSReadFileRaw(fd) + if err == nil { + sum := sha256.Sum256(data) + file.SHA256 = hex.EncodeToString(sum[:]) + file.Size = len(data) + if *ingestMode&1 == 1 { + if fd.Type() == disk.FileTypeAPP { + file.Text = disk.ApplesoftDetoks(data) + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.Data = data + file.LoadAddress = 0x801 + } else if fd.Type() == disk.FileTypeINT { + file.Text = disk.IntegerDetoks(data) + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.LoadAddress = 0x1000 + file.Data = data + } else if fd.Type() == disk.FileTypeTXT { + file.Text = disk.StripText(data) + file.Data = data + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + file.LoadAddress = 0x0000 + } else if fd.Type() == disk.FileTypeBIN && len(data) >= 2 { + file.LoadAddress = int(data[0]) + 256*int(data[1]) + file.Data = data[2:] + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + } else { + file.LoadAddress = 0x0000 + file.Data = data + file.TypeCode = TypeMask_AppleDOS | TypeCode(fd.Type()) + } + } + } + //l.Log("end read") + + info.Files = append(info.Files, &file) + + } + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) +} diff --git a/drvgeneric.go b/drvgeneric.go new file mode 100644 index 0000000..b727323 --- /dev/null +++ b/drvgeneric.go @@ -0,0 +1,86 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzeNONE(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + switch len(dsk.Data) { + case disk.STD_DISK_BYTES: + info.Tracks = 35 + info.Sectors = 16 + case disk.STD_DISK_BYTES_OLD: + info.Tracks = 35 + info.Sectors = 13 + case disk.PRODOS_800KB_DISK_BYTES: + info.Tracks = disk.GetDiskFormat(disk.DF_PRODOS_800KB).TPD() + info.Sectors = disk.GetDiskFormat(disk.DF_PRODOS_800KB).SPT() + default: + l.Errorf("Unknown size %d bytes", len(dsk.Data)) + } + + l.Logf("Tracks: %d, Sectors: %d", info.Tracks, info.Sectors) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + l.Logf("Assuming all sectors might be used") + info.Bitmap = make([]bool, info.Tracks*info.Sectors) + for i := range info.Bitmap { + info.Bitmap[i] = true + } + + info.ActiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + for t := 0; t < info.Tracks; t++ { + + for s := 0; s < info.Sectors; s++ { + + if info.Bitmap[t*info.Sectors+s] { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + activeData = append(activeData, data...) + + if *ingestMode&2 == 2 { + sector.Data = data + } + + info.ActiveSectors = append(info.ActiveSectors, sector) + } + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // Analyzing files + l.Log("Skipping Analysis of files") + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) + +} diff --git a/drvpascal.go b/drvpascal.go new file mode 100644 index 0000000..b3f3e46 --- /dev/null +++ b/drvpascal.go @@ -0,0 +1,176 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "time" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func dump(in []byte) string { + out := "" + for i, v := range in { + if i%16 == 0 { + if out != "" { + out += "\n" + } + out += fmt.Sprintf("%.4x: ", i) + } + out += fmt.Sprintf("%.2x ", v) + } + out += "\n" + return out +} + +func analyzePASCAL(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + l.Logf("Reading Disk Structure...") + + info.Blocks = dsk.Format.BPD() + + l.Logf("Blocks: %d", info.Blocks) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Tracks*info.Sectors) + + info.ActiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + var err error + info.Bitmap, err = dsk.PascalUsedBitmap() + if err != nil { + l.Errorf("Error reading bitmap: %s", err.Error()) + return + } + + for b := 0; b < info.Blocks; b++ { + + if info.Bitmap[b] { + + data, _ := dsk.PRODOSGetBlock(b) + + t, s1, s2 := dsk.PRODOSGetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.ActiveSectors = append(info.ActiveSectors, sec1, sec2) + + activeData = append(activeData, data...) + + } else { + + data, _ := dsk.PRODOSGetBlock(b) + + t, s1, s2 := dsk.PRODOSGetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.InactiveSectors = append(info.InactiveSectors, sec1, sec2) + + //activeData = append(activeData, data...) + + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // Analyzing files + l.Log("Starting Analysis of files") + + files, err := dsk.PascalGetCatalog("*") + if err != nil { + l.Errorf("Problem reading directory: %s", err.Error()) + return + } + + info.Files = make([]*DiskFile, 0) + for _, fd := range files { + l.Logf("- Name=%s, Type=%d, Len=%d", fd.GetName(), fd.GetType(), fd.GetFileSize()) + + file := DiskFile{ + Filename: fd.GetName(), + Type: fd.GetType().String(), + Locked: fd.IsLocked(), + Ext: fd.GetType().Ext(), + Created: time.Now(), + Modified: time.Now(), + } + + //l.Log("start read") + data, err := dsk.PascalReadFile(fd) + if err == nil { + sum := sha256.Sum256(data) + file.SHA256 = hex.EncodeToString(sum[:]) + file.Size = len(data) + if *ingestMode&1 == 1 { + // text ingestion + if fd.GetType() == disk.FileType_PAS_TEXT { + file.Text = disk.StripText(data) + file.Data = data + file.TypeCode = TypeMask_Pascal | TypeCode(fd.GetType()) + } else { + file.Data = data + file.TypeCode = TypeMask_Pascal | TypeCode(fd.GetType()) + } + } + } + + //l.Log("end read") + + info.Files = append(info.Files, &file) + + } + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) + +} diff --git a/drvprodos16.go b/drvprodos16.go new file mode 100644 index 0000000..671e7e3 --- /dev/null +++ b/drvprodos16.go @@ -0,0 +1,224 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzePRODOS16(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + isPD, Format, Layout := dsk.IsProDOS() + l.Logf("IsProDOS=%v, Format=%s, Layout=%d", isPD, Format, Layout) + if isPD { + dsk.Layout = Layout + } + + // Sector bitmap + l.Logf("Reading Disk VTOC...") + vtoc, err := dsk.PRODOSGetVDH(2) + if err != nil { + l.Errorf("Error reading VTOC: %s", err.Error()) + return + } + + info.Blocks = vtoc.GetTotalBlocks() + + l.Logf("Filecount: %d", vtoc.GetFileCount()) + + l.Logf("Blocks: %d", info.Blocks) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Blocks) + + info.ActiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + vbitmap, err := dsk.PRODOSGetVolumeBitmap() + if err != nil { + l.Errorf("Error reading volume bitmap: %s", err.Error()) + return + } + + l.Debug(vbitmap) + + for b := 0; b < info.Blocks; b++ { + info.Bitmap[b] = !vbitmap.IsBlockFree(b) + + if info.Bitmap[b] { + + data, _ := dsk.PRODOSGetBlock(b) + + t, s1, s2 := dsk.PRODOSGetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.ActiveSectors = append(info.ActiveSectors, sec1, sec2) + + activeData = append(activeData, data...) + + } else { + + data, _ := dsk.PRODOSGetBlock(b) + + t, s1, s2 := dsk.PRODOSGetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.InactiveSectors = append(info.InactiveSectors, sec1, sec2) + + //activeData = append(activeData, data...) + + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // // Analyzing files + l.Log("Starting Analysis of files") + + prodosDir(id, 2, "", dsk, info) + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + e := info.WriteToFile(*baseName + "/" + info.GetFilename()) + if e != nil { + l.Errorf("Error writing fingerprint: %v", e) + panic(e) + } + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) + +} + +func prodosDir(id int, start int, path string, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + _, files, err := dsk.PRODOSGetCatalog(start, "*") + if err != nil { + l.Errorf("Problem reading directory: %s", err.Error()) + return + } + if info.Files == nil { + info.Files = make([]*DiskFile, 0) + } + for _, fd := range files { + l.Logf("- Path=%s, Name=%s, Type=%s", path, fd.NameUnadorned(), fd.Type()) + + var file DiskFile + + if path == "" { + + file = DiskFile{ + Filename: fd.NameUnadorned(), + Type: fd.Type().String(), + Locked: fd.IsLocked(), + Ext: fd.Type().Ext(), + Created: fd.CreateTime(), + Modified: fd.ModTime(), + } + + } else { + + file = DiskFile{ + Filename: path + "/" + fd.NameUnadorned(), + Type: fd.Type().String(), + Locked: fd.IsLocked(), + Ext: fd.Type().Ext(), + Created: fd.CreateTime(), + Modified: fd.ModTime(), + } + + } + + if fd.Type() != disk.FileType_PD_Directory { + _, _, data, err := dsk.PRODOSReadFileRaw(fd) + if err == nil { + sum := sha256.Sum256(data) + file.SHA256 = hex.EncodeToString(sum[:]) + file.Size = len(data) + if *ingestMode&1 == 1 { + if fd.Type() == disk.FileType_PD_APP { + file.Text = disk.ApplesoftDetoks(data) + file.TypeCode = TypeMask_ProDOS | TypeCode(fd.Type()) + file.Data = data + file.LoadAddress = fd.AuxType() + } else if fd.Type() == disk.FileType_PD_INT { + file.Text = disk.IntegerDetoks(data) + file.TypeCode = TypeMask_ProDOS | TypeCode(fd.Type()) + file.Data = data + file.LoadAddress = fd.AuxType() + } else if fd.Type() == disk.FileType_PD_TXT { + file.Text = disk.StripText(data) + file.Data = data + file.TypeCode = TypeMask_ProDOS | TypeCode(fd.Type()) + file.LoadAddress = fd.AuxType() + } else { + file.LoadAddress = fd.AuxType() + file.Data = data + file.TypeCode = TypeMask_ProDOS | TypeCode(fd.Type()) + } + } + } + } + + info.Files = append(info.Files, &file) + + if fd.Type() == disk.FileType_PD_Directory { + newpath := path + if path != "" { + newpath += "/" + fd.NameUnadorned() + } else { + newpath = fd.NameUnadorned() + } + prodosDir(id, fd.IndexBlock(), newpath, dsk, info) + } + + } + +} diff --git a/drvprodos800.go b/drvprodos800.go new file mode 100644 index 0000000..b79554c --- /dev/null +++ b/drvprodos800.go @@ -0,0 +1,124 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzePRODOS800(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + l.Logf("Reading Disk VTOC...") + vtoc, err := dsk.PRODOS800GetVDH(2) + if err != nil { + l.Errorf("Error reading VTOC: %s", err.Error()) + return + } + + info.Blocks = vtoc.GetTotalBlocks() + l.Logf("Blocks: %d", info.Blocks) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Blocks) + + info.ActiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + vbitmap, err := dsk.PRODOS800GetVolumeBitmap() + if err != nil { + l.Errorf("Error reading volume bitmap: %s", err.Error()) + return + } + + l.Debug(vbitmap) + + for b := 0; b < info.Blocks; b++ { + info.Bitmap[b] = !vbitmap.IsBlockFree(b) + + if info.Bitmap[b] { + + data, _ := dsk.PRODOS800GetBlock(b) + + t, s1, s2 := dsk.PRODOS800GetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.ActiveSectors = append(info.ActiveSectors, sec1, sec2) + + activeData = append(activeData, data...) + + } else { + + data, _ := dsk.PRODOS800GetBlock(b) + + t, s1, s2 := dsk.PRODOS800GetBlockSectors(b) + + sec1 := &DiskSector{ + Track: t, + Sector: s1, + SHA256: dsk.ChecksumSector(t, s1), + } + + sec2 := &DiskSector{ + Track: t, + Sector: s2, + SHA256: dsk.ChecksumSector(t, s2), + } + + if *ingestMode&2 == 2 { + sec1.Data = data[:256] + sec2.Data = data[256:] + } + + info.InactiveSectors = append(info.InactiveSectors, sec1, sec2) + + //activeData = append(activeData, data...) + + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // // Analyzing files + l.Log("Starting Analysis of files") + + info.Files = make([]*DiskFile, 0) + prodosDir(id, 2, "", dsk, info) + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) + +} diff --git a/drvrdos.go b/drvrdos.go new file mode 100644 index 0000000..5a3431e --- /dev/null +++ b/drvrdos.go @@ -0,0 +1,142 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" +) + +func analyzeRDOS(id int, dsk *disk.DSKWrapper, info *Disk) { + + l := loggy.Get(id) + + // Sector bitmap + l.Logf("Reading Disk Structure...") + + info.Tracks, info.Sectors = 35, dsk.RDOSFormat.Spec().SectorMax + + l.Logf("Tracks: %d, Sectors: %d", info.Tracks, info.Sectors) + + l.Logf("Reading sector bitmap and SHA256'ing sectors") + + info.Bitmap = make([]bool, info.Tracks*info.Sectors) + + info.ActiveSectors = make(DiskSectors, 0) + info.InactiveSectors = make(DiskSectors, 0) + + activeData := make([]byte, 0) + + var err error + info.Bitmap, err = dsk.RDOSUsedBitmap() + if err != nil { + l.Errorf("Error reading bitmap: %s", err.Error()) + return + } + + for t := 0; t < info.Tracks; t++ { + + for s := 0; s < info.Sectors; s++ { + + if info.Bitmap[t*info.Sectors+s] { + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + activeData = append(activeData, data...) + + if *ingestMode&2 == 2 { + sector.Data = data + } + + info.ActiveSectors = append(info.ActiveSectors, sector) + } else { + + sector := &DiskSector{ + Track: t, + Sector: s, + SHA256: dsk.ChecksumSector(t, s), + } + + data := dsk.Read() + if *ingestMode&2 == 2 { + sector.Data = data + } + //activeData = append(activeData, data...) + + info.InactiveSectors = append(info.InactiveSectors, sector) + + } + } + + } + + sum := sha256.Sum256(activeData) + info.SHA256Active = hex.EncodeToString(sum[:]) + + info.LogBitmap(id) + + // Analyzing files + l.Log("Starting Analysis of files") + + files, err := dsk.RDOSGetCatalog("*") + if err != nil { + l.Errorf("Problem reading directory: %s", err.Error()) + return + } + + info.Files = make([]*DiskFile, 0) + for _, fd := range files { + l.Logf("- Name=%s, Type=%s", fd.NameUnadorned(), fd.Type()) + + file := DiskFile{ + Filename: fd.NameUnadorned(), + Type: fd.Type().String(), + Ext: fd.Type().Ext(), + } + + //l.Log("start read") + data, err := dsk.RDOSReadFile(fd) + if err == nil { + sum := sha256.Sum256(data) + file.SHA256 = hex.EncodeToString(sum[:]) + file.Size = len(data) + if *ingestMode&1 == 1 { + if fd.Type() == disk.FileType_RDOS_AppleSoft { + file.Text = disk.ApplesoftDetoks(data) + file.TypeCode = TypeMask_RDOS | TypeCode(fd.Type()) + file.Data = data + } else if fd.Type() == disk.FileType_RDOS_Text { + file.Text = disk.StripText(data) + file.TypeCode = TypeMask_RDOS | TypeCode(fd.Type()) + file.Data = data + } else { + file.Data = data + file.LoadAddress = fd.LoadAddress() + file.TypeCode = TypeMask_RDOS | TypeCode(fd.Type()) + } + } + } + //l.Log("end read") + + l.Logf("FILETEXT=\n%s", dump(data)) + + info.Files = append(info.Files, &file) + + } + + exists := exists(*baseName + "/" + info.GetFilename()) + + if !exists || *forceIngest { + info.WriteToFile(*baseName + "/" + info.GetFilename()) + } else { + l.Log("Not writing as it already exists") + } + + out(dsk.Format) + +} diff --git a/fuzzyblocks.go b/fuzzyblocks.go new file mode 100644 index 0000000..a98f067 --- /dev/null +++ b/fuzzyblocks.go @@ -0,0 +1,410 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "sync" +) + +const EMPTYSECTOR = "5341e6b2646979a70e57653007a1f310169421ec9bdd9f1a5648f75ade005af1" + +func GetAllDiskSectors(pattern string, pathfilter []string) map[string]DiskSectors { + + cache := make(map[string]DiskSectors) + + exists, matches := existsPattern(*baseName, pathfilter, pattern) + if !exists { + return cache + } + + workchan := make(chan string, 100) + var s sync.Mutex + var wg sync.WaitGroup + + for i := 0; i < ingestWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + item := &Disk{} + if err := item.ReadFromFile(m); err == nil { + + // + chunk := append(item.ActiveSectors, item.InactiveSectors...) + tmp := make(DiskSectors, 0) + for _, v := range chunk { + if v.SHA256 != EMPTYSECTOR { + tmp = append(tmp, v) + } else { + fmt.Printf("%s: throw away zero sector T%d,S%d\n", item.Filename, v.Track, v.Sector) + } + } + + // Load cache + s.Lock() + cache[item.FullPath] = tmp + s.Unlock() + + } + } + wg.Done() + }() + } + + var lastPc int = -1 + for i, m := range matches { + + workchan <- m + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Caching disk sector data... %d%% ", pc)) + } + + lastPc = pc + } + close(workchan) + + wg.Wait() + + return cache +} + +func GetActiveDiskSectors(pattern string, pathfilter []string) map[string]DiskSectors { + + cache := make(map[string]DiskSectors) + + exists, matches := existsPattern(*baseName, pathfilter, pattern) + if !exists { + return cache + } + + workchan := make(chan string, 100) + var s sync.Mutex + var wg sync.WaitGroup + + for i := 0; i < ingestWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + item := &Disk{} + if err := item.ReadFromFile(m); err == nil { + + // Load cache + s.Lock() + cache[item.FullPath] = item.ActiveSectors + s.Unlock() + + } + } + wg.Done() + }() + } + + var lastPc int = -1 + for i, m := range matches { + + workchan <- m + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Caching disk sector data... %d%% ", pc)) + } + + lastPc = pc + } + close(workchan) + + wg.Wait() + + return cache +} + +func GetSectorMap(d DiskSectors) map[string]*DiskSector { + + out := make(map[string]*DiskSector) + for _, v := range d { + out[fmt.Sprintf("T%d,S%d", v.Track, v.Sector)] = v + } + return out + +} + +type SectorOverlapRecord struct { + same map[string]map[*DiskSector]*DiskSector + percent map[string]float64 + missing map[string][]*DiskSector + extras map[string][]*DiskSector +} + +func (f *SectorOverlapRecord) Remove(key string) { + delete(f.same, key) + delete(f.percent, key) + delete(f.missing, key) + delete(f.extras, key) +} + +func (f *SectorOverlapRecord) IsSubsetOf(filename string) bool { + + // f is a subset if: + // missing == 0 + // extra > 0 + + if _, ok := f.same[filename]; !ok { + return false + } + + return len(f.extras[filename]) > 0 && len(f.missing[filename]) == 0 + +} + +func (f *SectorOverlapRecord) IsSupersetOf(filename string) bool { + + // f is a superset if: + // missing > 0 + // extra == 0 + + if _, ok := f.same[filename]; !ok { + return false + } + + return len(f.extras[filename]) == 0 && len(f.missing[filename]) > 0 + +} + +func CompareSectors(d, b DiskSectors, r *SectorOverlapRecord, key string) float64 { + + var sameSectors float64 + var missingSectors float64 + var extraSectors float64 + + var dmap = GetSectorMap(d) + var bmap = GetSectorMap(b) + + for fileCk, info := range dmap { + + binfo, bEx := bmap[fileCk] + + if bEx && info.SHA256 == binfo.SHA256 { + sameSectors += 1 + if r.same[key] == nil { + r.same[key] = make(map[*DiskSector]*DiskSector) + } + + r.same[key][binfo] = info + } else { + missingSectors += 1 + if r.missing[key] == nil { + r.missing[key] = make([]*DiskSector, 0) + } + r.missing[key] = append(r.missing[key], info) + } + + } + + for fileCk, info := range bmap { + + _, dEx := dmap[fileCk] + + if !dEx { + extraSectors += 1 + // file match + if r.extras[key] == nil { + r.extras[key] = make([]*DiskSector, 0) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + r.extras[key] = append(r.extras[key], info) + } + + } + + if (sameSectors + extraSectors + missingSectors) == 0 { + return 0 + } + + // return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal + return sameSectors / (sameSectors + extraSectors + missingSectors) + +} + +// Actual fuzzy file match report +func CollectSectorOverlapsAboveThreshold(t float64, pathfilter []string, ff func(pattern string, pathfilter []string) map[string]DiskSectors) map[string]*SectorOverlapRecord { + + filerecords := ff("*_*_*_*.fgp", pathfilter) + + results := make(map[string]*SectorOverlapRecord) + + workchan := make(chan string, 100) + var wg sync.WaitGroup + var s sync.Mutex + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + for i := 0; i < processWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + + v := &SectorOverlapRecord{ + same: make(map[string]map[*DiskSector]*DiskSector), + percent: make(map[string]float64), + missing: make(map[string][]*DiskSector), + extras: make(map[string][]*DiskSector), + } + + d := filerecords[m] + + for k, b := range filerecords { + if k == m { + continue // dont compare ourselves + } + // ok good to compare -- only keep if we need our threshold + + if closeness := CompareSectors(d, b, v, k); closeness < t { + v.Remove(k) + } else { + v.percent[k] = closeness + } + } + + // since we delete < threshold, only add if we have any result + if len(v.percent) > 0 { + //os.Stderr.WriteString("\r\nAdded file: " + m + "\r\n\r\n") + s.Lock() + results[m] = v + s.Unlock() + } + + } + wg.Done() + }() + } + + // feed data in + var lastPc int = -1 + var i int + for k, _ := range filerecords { + + if len(c) > 0 { + sig := <-c + if sig == os.Interrupt { + close(c) + os.Stderr.WriteString("\r\nInterrupted. Waiting for workers to stop.\r\n\r\n") + break + } + } + + workchan <- k + + pc := int(100 * float64(i) / float64(len(filerecords))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Processing sectors data... %d%% ", pc)) + } + + lastPc = pc + i++ + } + + close(workchan) + wg.Wait() + + return results + +} + +// Actual fuzzy file match report +func CollectSectorSubsets(pathfilter []string, ff func(pattern string, pathfilter []string) map[string]DiskSectors) map[string]*SectorOverlapRecord { + + filerecords := ff("*_*_*_*.fgp", pathfilter) + + results := make(map[string]*SectorOverlapRecord) + + workchan := make(chan string, 100) + var wg sync.WaitGroup + var s sync.Mutex + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + for i := 0; i < processWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + + v := &SectorOverlapRecord{ + same: make(map[string]map[*DiskSector]*DiskSector), + percent: make(map[string]float64), + missing: make(map[string][]*DiskSector), + extras: make(map[string][]*DiskSector), + } + + d := filerecords[m] + + for k, b := range filerecords { + if k == m { + continue // dont compare ourselves + } + // ok good to compare -- only keep if we need our threshold + + closeness := CompareSectors(d, b, v, k) + + if !v.IsSubsetOf(k) { + v.Remove(k) + } else { + v.percent[k] = closeness + } + } + + // since we delete < threshold, only add if we have any result + if len(v.percent) > 0 { + //os.Stderr.WriteString("\r\nAdded file: " + m + "\r\n\r\n") + s.Lock() + results[m] = v + s.Unlock() + } + + } + wg.Done() + }() + } + + // feed data in + var lastPc int = -1 + var i int + for k, _ := range filerecords { + + if len(c) > 0 { + sig := <-c + if sig == os.Interrupt { + close(c) + os.Stderr.WriteString("\r\nInterrupted. Waiting for workers to stop.\r\n\r\n") + break + } + } + + workchan <- k + + pc := int(100 * float64(i) / float64(len(filerecords))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Processing sectors data... %d%% ", pc)) + } + + lastPc = pc + i++ + } + + close(workchan) + wg.Wait() + + return results + +} diff --git a/fuzzyfiles.go b/fuzzyfiles.go new file mode 100644 index 0000000..beb52b7 --- /dev/null +++ b/fuzzyfiles.go @@ -0,0 +1,467 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "strings" + "sync" +) + +func inList(item string, list []string) bool { + for _, v := range list { + if strings.ToLower(v) == strings.ToLower(item) { + return true + } + } + return false +} + +const EXCLUDEZEROBYTE = true +const EXCLUDEHELLO = true + +func GetAllFiles(pattern string, pathfilter []string) map[string]DiskCatalog { + + cache := make(map[string]DiskCatalog) + + exists, matches := existsPattern(*baseName, pathfilter, pattern) + if !exists { + return cache + } + + workchan := make(chan string, 100) + var s sync.Mutex + var wg sync.WaitGroup + + for i := 0; i < ingestWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + item := &Disk{} + if err := item.ReadFromFile(m); err == nil { + + if len(item.Files) == 0 { + continue + } + + // Load cache + s.Lock() + cache[item.FullPath] = item.Files + s.Unlock() + + } else { + fmt.Println("FAIL") + } + } + wg.Done() + }() + } + + var lastPc int = -1 + for i, m := range matches { + + //fmt.Printf("Queue: %s\n", m) + + workchan <- m + + pc := int(100 * float64(i) / float64(len(matches))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Caching data... %d%% ", pc)) + } + + lastPc = pc + } + close(workchan) + + wg.Wait() + + return cache +} + +type FileOverlapRecord struct { + files map[string]map[*DiskFile]*DiskFile + percent map[string]float64 + missing map[string][]*DiskFile + extras map[string][]*DiskFile +} + +func (f *FileOverlapRecord) Remove(key string) { + delete(f.files, key) + delete(f.percent, key) + delete(f.missing, key) + delete(f.extras, key) +} + +func (f *FileOverlapRecord) IsSubsetOf(filename string) bool { + + // f is a subset if: + // missing == 0 + // extra > 0 + + if _, ok := f.files[filename]; !ok { + return false + } + + return len(f.extras[filename]) > 0 && len(f.missing[filename]) == 0 + +} + +func (f *FileOverlapRecord) IsSupersetOf(filename string) bool { + + // f is a superset if: + // missing > 0 + // extra == 0 + + if _, ok := f.files[filename]; !ok { + return false + } + + return len(f.extras[filename]) == 0 && len(f.missing[filename]) > 0 + +} + +// Actual fuzzy file match report +func CollectFilesOverlapsAboveThreshold(t float64, pathfilter []string) map[string]*FileOverlapRecord { + + filerecords := GetAllFiles("*_*_*_*.fgp", pathfilter) + + results := make(map[string]*FileOverlapRecord) + + workchan := make(chan string, 100) + var wg sync.WaitGroup + var s sync.Mutex + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + for i := 0; i < processWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + + v := &FileOverlapRecord{ + files: make(map[string]map[*DiskFile]*DiskFile), + percent: make(map[string]float64), + missing: make(map[string][]*DiskFile), + extras: make(map[string][]*DiskFile), + } + + d := filerecords[m] + + for k, b := range filerecords { + if k == m { + continue // dont compare ourselves + } + // ok good to compare -- only keep if we need our threshold + + if closeness := CompareCatalogs(d, b, v, k); closeness < t { + v.Remove(k) + } else { + v.percent[k] = closeness + } + } + + // since we delete < threshold, only add if we have any result + if len(v.percent) > 0 { + //os.Stderr.WriteString("\r\nAdded file: " + m + "\r\n\r\n") + s.Lock() + results[m] = v + s.Unlock() + } + + } + wg.Done() + }() + } + + // feed data in + var lastPc int = -1 + var i int + for k, _ := range filerecords { + + if len(c) > 0 { + sig := <-c + if sig == os.Interrupt { + close(c) + os.Stderr.WriteString("\r\nInterrupted. Waiting for workers to stop.\r\n\r\n") + break + } + } + + workchan <- k + + pc := int(100 * float64(i) / float64(len(filerecords))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Processing files data... %d%% ", pc)) + } + + lastPc = pc + i++ + } + + close(workchan) + wg.Wait() + + return results + +} + +func GetCatalogMap(d DiskCatalog) map[string]*DiskFile { + + out := make(map[string]*DiskFile) + for _, v := range d { + out[v.SHA256] = v + } + return out + +} + +func CompareCatalogs(d, b DiskCatalog, r *FileOverlapRecord, key string) float64 { + + var sameFiles float64 + var missingFiles float64 + var extraFiles float64 + + var dmap = GetCatalogMap(d) + var bmap = GetCatalogMap(b) + + for fileCk, info := range dmap { + + if info.Size == 0 && EXCLUDEZEROBYTE { + continue + } + + if info.Filename == "hello" && EXCLUDEHELLO { + continue + } + + binfo, bEx := bmap[fileCk] + + if bEx { + sameFiles += 1 + // file match + if r.files[key] == nil { + r.files[key] = make(map[*DiskFile]*DiskFile) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + r.files[key][binfo] = info + } else { + missingFiles += 1 + // file match + if r.missing[key] == nil { + r.missing[key] = make([]*DiskFile, 0) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + r.missing[key] = append(r.missing[key], info) + } + + } + + for fileCk, info := range bmap { + + if info.Size == 0 { + continue + } + + _, dEx := dmap[fileCk] + + if !dEx { + extraFiles += 1 + // file match + if r.extras[key] == nil { + r.extras[key] = make([]*DiskFile, 0) + } + //fmt.Printf("*** %s: %s -> %s\n", b.Filename, binfo.Filename, info.Filename) + r.extras[key] = append(r.extras[key], info) + } + + } + + if (sameFiles + extraFiles + missingFiles) == 0 { + return 0 + } + + // return sameSectors / dTotal, sameSectors / bTotal, diffSectors / dTotal, diffSectors / btotal + return sameFiles / (sameFiles + extraFiles + missingFiles) + +} + +func CollectFileSubsets(pathfilter []string) map[string]*FileOverlapRecord { + + filerecords := GetAllFiles("*_*_*_*.fgp", pathfilter) + + results := make(map[string]*FileOverlapRecord) + + workchan := make(chan string, 100) + var wg sync.WaitGroup + var s sync.Mutex + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + for i := 0; i < processWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + + v := &FileOverlapRecord{ + files: make(map[string]map[*DiskFile]*DiskFile), + percent: make(map[string]float64), + missing: make(map[string][]*DiskFile), + extras: make(map[string][]*DiskFile), + } + + d := filerecords[m] + + for k, b := range filerecords { + if k == m { + continue // dont compare ourselves + } + // ok good to compare -- only keep if we need our threshold + + closeness := CompareCatalogs(d, b, v, k) + if !v.IsSubsetOf(k) { + v.Remove(k) + } else { + v.percent[k] = closeness + } + } + + // since we delete < threshold, only add if we have any result + if len(v.percent) > 0 { + //os.Stderr.WriteString("\r\nAdded file: " + m + "\r\n\r\n") + s.Lock() + results[m] = v + s.Unlock() + } + + } + wg.Done() + }() + } + + // feed data in + var lastPc int = -1 + var i int + for k, _ := range filerecords { + + if len(c) > 0 { + sig := <-c + if sig == os.Interrupt { + close(c) + os.Stderr.WriteString("\r\nInterrupted. Waiting for workers to stop.\r\n\r\n") + break + } + } + + workchan <- k + + pc := int(100 * float64(i) / float64(len(filerecords))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Processing files data... %d%% ", pc)) + } + + lastPc = pc + i++ + } + + close(workchan) + wg.Wait() + + return results + +} + +func CollectFilesOverlapsCustom(keep func(d1, d2 string, v *FileOverlapRecord) bool, pathfilter []string) map[string]*FileOverlapRecord { + + filerecords := GetAllFiles("*_*_*_*.fgp", pathfilter) + + results := make(map[string]*FileOverlapRecord) + + workchan := make(chan string, 100) + var wg sync.WaitGroup + var s sync.Mutex + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + for i := 0; i < processWorkers; i++ { + wg.Add(1) + go func() { + for m := range workchan { + + v := &FileOverlapRecord{ + files: make(map[string]map[*DiskFile]*DiskFile), + percent: make(map[string]float64), + missing: make(map[string][]*DiskFile), + extras: make(map[string][]*DiskFile), + } + + d := filerecords[m] + + for k, b := range filerecords { + if k == m { + continue // dont compare ourselves + } + // ok good to compare -- only keep if we need our threshold + closeness := CompareCatalogs(d, b, v, k) + + if !keep(m, k, v) { + v.Remove(k) + } else { + v.percent[k] = closeness + } + } + + // since we delete < threshold, only add if we have any result + if len(v.files) > 0 { + //os.Stderr.WriteString("\r\nAdded file: " + m + "\r\n\r\n") + s.Lock() + results[m] = v + s.Unlock() + } + + } + wg.Done() + }() + } + + // feed data in + var lastPc int = -1 + var i int + for k, _ := range filerecords { + + if len(c) > 0 { + sig := <-c + if sig == os.Interrupt { + close(c) + os.Stderr.WriteString("\r\nInterrupted. Waiting for workers to stop.\r\n\r\n") + break + } + } + + workchan <- k + + pc := int(100 * float64(i) / float64(len(filerecords))) + + if pc != lastPc { + fmt.Print("\r") + os.Stderr.WriteString(fmt.Sprintf("Processing files data... %d%% ", pc)) + } + + lastPc = pc + i++ + } + + close(workchan) + wg.Wait() + + return results + +} diff --git a/ingestor.go b/ingestor.go new file mode 100644 index 0000000..98746e4 --- /dev/null +++ b/ingestor.go @@ -0,0 +1,358 @@ +package main + +import ( + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "runtime/debug" + "sync" + "time" + + "strings" + + "crypto/md5" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" + "github.com/paleotronic/diskm8/panic" +) + +var diskRegex = regexp.MustCompile("(?i)[.](po|do|dsk)$") + +func processFile(path string, info os.FileInfo, err error) error { + if err != nil { + loggy.Get(0).Errorf(err.Error()) + return err + } + + if diskRegex.MatchString(path) { + + incoming <- path + + fmt.Printf("\rIngested: %d volumes ...", processed) + + } + + return nil +} + +const loaderWorkers = 8 + +var incoming chan string +var processed int +var errorcount int +var indisk map[disk.DiskFormat]int +var outdisk map[disk.DiskFormat]int +var cm sync.Mutex + +func init() { + indisk = make(map[disk.DiskFormat]int) + outdisk = make(map[disk.DiskFormat]int) +} + +func in(f disk.DiskFormat) { + cm.Lock() + indisk[f] = indisk[f] + 1 + cm.Unlock() +} + +func out(f disk.DiskFormat) { + cm.Lock() + outdisk[f] = outdisk[f] + 1 + cm.Unlock() +} + +func walk(dir string) { + + start := time.Now() + + incoming = make(chan string, 16) + indisk = make(map[disk.DiskFormat]int) + outdisk = make(map[disk.DiskFormat]int) + + var wg sync.WaitGroup + var s sync.Mutex + + for i := 0; i < loaderWorkers; i++ { + wg.Add(1) + go func(i int) { + + id := 1 + i + l := loggy.Get(id) + + for filename := range incoming { + + panic.Do( + func() { + analyze(id, filename) + s.Lock() + processed++ + s.Unlock() + }, + func(r interface{}) { + l.Errorf("Error processing volume: %s", filename) + l.Errorf(string(debug.Stack())) + s.Lock() + errorcount++ + s.Unlock() + }, + ) + + } + + wg.Done() + + }(i) + } + + filepath.Walk(dir, processFile) + + close(incoming) + wg.Wait() + + fmt.Printf("\rIngested: %d volumes ...", processed) + + fmt.Println() + + duration := time.Since(start) + + fmt.Println("=============================================================") + fmt.Printf(" DSKalyzer process report (%d Workers, %v)\n", loaderWorkers, duration) + fmt.Println("=============================================================") + + tin, tout := 0, 0 + + for f, count := range indisk { + outcount := outdisk[f] + fmt.Printf("%-30s %6d in %6d out\n", f.String(), count, outcount) + tin += count + tout += outcount + } + + fmt.Println() + + fmt.Printf("%-30s %6d in %6d out\n", "Total", tin, tout) + + fmt.Println() + + average := duration / time.Duration(processed+errorcount) + + fmt.Printf("%v average time spent per disk.\n", average) +} + +func existsPatternOld(base string, pattern string) (bool, []string) { + + l := loggy.Get(0) + + p := base + "/" + pattern + + l.Logf("glob: %s", p) + + matches, _ := filepath.Glob(p) + + return (len(matches) > 0), matches + +} + +func resolvePathfilters(base string, pathfilter []string, pattern string) []*regexp.Regexp { + + tmp := strings.Replace(pattern, ".", "[.]", -1) + tmp = strings.Replace(tmp, "?", ".", -1) + tmp = strings.Replace(tmp, "*", ".+", -1) + tmp += "$" + + // pathfilter either contains filenames or a pattern (eg. if it was quoted) + var out []*regexp.Regexp + for _, p := range pathfilter { + + if runtime.GOOS == "windows" { + //p = strings.Replace(p, ":", "", -1) + p = strings.Replace(p, "\\", "/", -1) + } + + //fmt.Printf("Stat [%s]\n", p) + + p, e := filepath.Abs(p) + if e != nil { + continue + } + + //fmt.Printf("OK\n") + + // path is okay and now absolute + info, e := os.Stat(p) + if e != nil { + continue + } + + if runtime.GOOS == "windows" { + p = strings.Replace(p, ":", "", -1) + p = strings.Replace(p, "\\", "/", -1) + } + + var realpath string + if info.IsDir() { + realpath = strings.Replace(base, "\\", "/", -1) + "/" + strings.Trim(p, "/") + "/" + tmp + } else { + // file + b := strings.Trim(filepath.Base(p), " /") + s := md5.Sum([]byte(b)) + realpath = strings.Replace(base, "\\", "/", -1) + "/" + strings.Trim(filepath.Dir(p), "/") + "/.+_.+_.+_" + hex.EncodeToString(s[:]) + "[.]fgp$" + } + + //fmt.Printf("Regexp [%s]\n", realpath) + + out = append(out, regexp.MustCompile(realpath)) + + } + + return out + +} + +func existsPattern(base string, filters []string, pattern string) (bool, []string) { + + tmp := strings.Replace(pattern, ".", "[.]", -1) + tmp = strings.Replace(tmp, "?", ".", -1) + tmp = strings.Replace(tmp, "*", ".+", -1) + tmp = "(?i)" + tmp + "$" + //os.Stderr.WriteString("Globby is: " + tmp + "\r\n") + fileRxp := regexp.MustCompile(tmp) + + var out []string + var found bool + + processPatternPath := func(path string, info os.FileInfo, err error) error { + + l := loggy.Get(0) + + if err != nil { + l.Errorf(err.Error()) + return err + } + + if fileRxp.MatchString(filepath.Base(path)) { + found = true + out = append(out, path) + } + + return nil + } + + filepath.Walk(base, processPatternPath) + + fexp := resolvePathfilters(base, filters, pattern) + + if len(fexp) > 0 { + out2 := make([]string, 0) + for _, p := range out { + + if runtime.GOOS == "windows" { + p = strings.Replace(p, "\\", "/", -1) + } + + for _, rxp := range fexp { + //fmt.Printf("Match [%s]\n", p) + if rxp.MatchString(p) { + out2 = append(out2, p) + //fmt.Printf("Match regexp: %s\n", p) + break + } + } + } + //fmt.Printf("%d returns\n", len(out2)) + return (len(out2) > 0), out2 + } + + //fmt.Printf("%d returns\n", len(out)) + + return found, out + +} + +func analyze(id int, filename string) (*Disk, error) { + + l := loggy.Get(id) + + var err error + var dsk *disk.DSKWrapper + var dskInfo Disk = Disk{} + + dskInfo.Filename = path.Base(filename) + + if abspath, e := filepath.Abs(filename); e == nil { + filename = abspath + } + + dskInfo.FullPath = path.Clean(filename) + + l.Logf("Reading disk image from file source %s", filename) + //fmt.Printf("Processing %s\n", filename) + //fmt.Print(".") + + dsk, err = disk.NewDSKWrapper(defNibbler, filename) + + if err != nil { + l.Errorf("Disk read failed: %s", err) + return &dskInfo, err + } + + if dsk.Format.ID == disk.DF_DOS_SECTORS_13 || dsk.Format.ID == disk.DF_DOS_SECTORS_16 { + isADOS, _, _ := dsk.IsAppleDOS() + if !isADOS { + dsk.Format.ID = disk.DF_NONE + dsk.Layout = disk.SectorOrderDOS33 + } + } + // fmt.Printf("%s: IsAppleDOS=%v, Format=%s, Layout=%d\n", path.Base(filename), isADOS, Format, Layout) + + l.Log("Load is OK.") + + dskInfo.SHA256 = dsk.ChecksumDisk() + l.Logf("SHA256 is %s", dskInfo.SHA256) + + dskInfo.Format = dsk.Format.String() + dskInfo.FormatID = dsk.Format + l.Logf("Format is %s", dskInfo.Format) + + l.Debugf("TOSO MAGIC: %v", hex.EncodeToString(dsk.Data[:32])) + + t, s := dsk.HuntVTOC(35, 13) + l.Logf("Hunt VTOC says: %d, %d", t, s) + + // Check if it exists + + in(dsk.Format) + + dskInfo.IngestMode = *ingestMode + + switch dsk.Format.ID { + case disk.DF_DOS_SECTORS_16: + analyzeDOS16(id, dsk, &dskInfo) + case disk.DF_DOS_SECTORS_13: + analyzeDOS13(id, dsk, &dskInfo) + case disk.DF_PRODOS_400KB: + analyzePRODOS800(id, dsk, &dskInfo) + case disk.DF_PRODOS_800KB: + analyzePRODOS800(id, dsk, &dskInfo) + case disk.DF_PRODOS: + analyzePRODOS16(id, dsk, &dskInfo) + case disk.DF_RDOS_3: + analyzeRDOS(id, dsk, &dskInfo) + case disk.DF_RDOS_32: + analyzeRDOS(id, dsk, &dskInfo) + case disk.DF_RDOS_33: + analyzeRDOS(id, dsk, &dskInfo) + case disk.DF_PASCAL: + analyzePASCAL(id, dsk, &dskInfo) + default: + analyzeNONE(id, dsk, &dskInfo) + } + + return &dskInfo, nil + +} diff --git a/loggy/logger.go b/loggy/logger.go new file mode 100644 index 0000000..040ee80 --- /dev/null +++ b/loggy/logger.go @@ -0,0 +1,138 @@ +package loggy + +import ( + "fmt" + "os" + "strings" + "time" +) + +var logFile *os.File +var ECHO bool = false +var SILENT bool = false +var LogFolder string = "./logs/" + +type Logger struct { + logFile *os.File + id int + app string +} + +var loggers map[int]*Logger +var app string + +func Get(id int) *Logger { + if loggers == nil { + loggers = make(map[int]*Logger) + } + l, ok := loggers[id] + if !ok { + l = NewLogger(id, app) + loggers[id] = l + } + return l +} + +func NewLogger(id int, app string) *Logger { + + if app == "" { + app = "dskalyzer" + } + + filename := fmt.Sprintf("%s_%d_%s.log", app, id, fts()) + os.MkdirAll(LogFolder, 0755) + + logFile, _ = os.Create(LogFolder + filename) + l := &Logger{ + id: id, + logFile: logFile, + app: app, + } + + return l +} + +func ts() string { + t := time.Now() + return fmt.Sprintf( + "%.4d/%.2d/%.2d %.2d:%.2d:%.2d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second(), + ) +} + +func fts() string { + t := time.Now() + return fmt.Sprintf( + "%.4d%.2d%.2d%.2d%.2d%.2d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second(), + ) +} + +func (l *Logger) llogf(format string, designator string, v ...interface{}) { + + format = ts() + " " + designator + " :: " + format + + if !strings.HasSuffix(format, "\n") { + format += "\n" + } + + l.logFile.WriteString(fmt.Sprintf(format, v...)) + l.logFile.Sync() + + if ECHO { + os.Stderr.WriteString(fmt.Sprintf(format, v...)) + } + +} + +func (l *Logger) llog(designator string, v ...interface{}) { + + format := ts() + " " + designator + " :: " + for _, vv := range v { + format += fmt.Sprintf("%v ", vv) + } + if !strings.HasSuffix(format, "\n") { + format += "\n" + } + + l.logFile.WriteString(format) + l.logFile.Sync() + + if ECHO { + os.Stderr.WriteString(format) + } +} + +func (l *Logger) Logf(format string, v ...interface{}) { + l.llogf(format, "INFO ", v...) +} + +func (l *Logger) Log(v ...interface{}) { + l.llog("INFO ", v...) +} + +func (l *Logger) Errorf(format string, v ...interface{}) { + l.llogf(format, "ERROR", v...) +} + +func (l *Logger) Error(v ...interface{}) { + l.llog("ERROR", v...) +} + +func (l *Logger) Debugf(format string, v ...interface{}) { + l.llogf(format, "DEBUG", v...) +} + +func (l *Logger) Debug(v ...interface{}) { + l.llog("DEBUG", v...) +} + +func (l *Logger) Fatalf(format string, v ...interface{}) { + l.llogf(format, "FATAL", v...) +} + +func (l *Logger) Fatal(v ...interface{}) { + l.llog("FATAL", v...) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..61da2ef --- /dev/null +++ b/main.go @@ -0,0 +1,352 @@ +package main + +/* +DiskM8 is an open source offshoot of the file handling code from the Octalyzer +project. + +It provides some command line tools for manipulating Apple // disk images, and +some work in progress reporting tools to ingest large quantities of files, +catalog them and detect duplicates. + +The code currently needs a lot of refactoring and cleanup, which we will be working +through as time goes by. +*/ + +import ( + "fmt" + "io/ioutil" + "path" + "path/filepath" + "runtime/debug" + + "flag" + + "os" + + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" + + "runtime" + + "strings" + + "time" + + "github.com/paleotronic/diskm8/panic" +) + +func usage() { + fmt.Printf(`%s + +Tool checks for duplicate or similar apple ][ disks, specifically those +with %d bytes size. + +`, path.Base(os.Args[0]), disk.STD_DISK_BYTES) + flag.PrintDefaults() +} + +func binpath() string { + + if runtime.GOOS == "windows" { + return os.Getenv("USERPROFILE") + "/DiskM8" + } + return os.Getenv("HOME") + "/DiskM8" + +} + +func init() { + loggy.LogFolder = binpath() + "/logs/" +} + +var dskName = flag.String("ingest", "", "Disk file or path to ingest") +var dskInfo = flag.String("query", "", "Disk file to query or analyze") +var baseName = flag.String("datastore", binpath()+"/fingerprints", "Database of disk fingerprints for checking") +var verbose = flag.Bool("verbose", false, "Log to stderr") +var fileDupes = flag.Bool("file-dupes", false, "Run file dupe report") +var wholeDupes = flag.Bool("whole-dupes", false, "Run whole disk dupe report") +var activeDupes = flag.Bool("as-dupes", false, "Run active sectors only disk dupe report") +var asPartial = flag.Bool("as-partial", false, "Run partial active sector match against single disk (-disk required)") +var similarity = flag.Float64("similarity", 0.90, "Object match threshold for -*-partial reports") +var minSame = flag.Int("min-same", 0, "Minimum same # files for -all-file-partial") +var maxDiff = flag.Int("max-diff", 0, "Maximum different # files for -all-file-partial") +var filePartial = flag.Bool("file-partial", false, "Run partial file match against single disk (-disk required)") +var fileMatch = flag.String("file", "", "Search for other disks containing file") +var dir = flag.Bool("dir", false, "Directory specified disk (needs -disk)") +var dirFormat = flag.String("dir-format", "{filename} {type} {size:kb} Checksum: {sha256}", "Format of dir") +var preCache = flag.Bool("c", true, "Cache data to memory for quicker processing") +var allFilePartial = flag.Bool("all-file-partial", false, "Run partial file match against all disks") +var allSectorPartial = flag.Bool("all-sector-partial", false, "Run partial sector match (all) against all disks") +var activeSectorPartial = flag.Bool("active-sector-partial", false, "Run partial sector match (active only) against all disks") +var allFileSubset = flag.Bool("all-file-subset", false, "Run subset file match against all disks") +var activeSectorSubset = flag.Bool("active-sector-subset", false, "Run subset (active) sector match against all disks") +var allSectorSubset = flag.Bool("all-sector-subset", false, "Run subset (non-zero) sector match against all disks") +var filterPath = flag.Bool("select", false, "Select files for analysis or search based on file/dir/mask") +var csvOut = flag.Bool("csv", false, "Output data to CSV format") +var reportFile = flag.String("out", "", "Output file (empty for stdout)") +var catDupes = flag.Bool("cat-dupes", false, "Run duplicate catalog report") +var searchFilename = flag.String("search-filename", "", "Search database for file with name") +var searchSHA = flag.String("search-sha", "", "Search database for file with checksum") +var searchTEXT = flag.String("search-text", "", "Search database for file containing text") +var forceIngest = flag.Bool("force", false, "Force re-ingest disks that already exist") +var ingestMode = flag.Int("ingest-mode", 1, "Ingest mode:\n\t0=Fingerprints only\n\t1=Fingerprints + text\n\t2=Fingerprints + sector data\n\t3=All") +var extract = flag.String("extract", "", "Extract files/disks matched in searches ('#'=extract disk, '@'=extract files)") +var adornedCP = flag.Bool("adorned", true, "Extract files named similar to CP") +var shell = flag.Bool("shell", false, "Start interactive mode") +var shellBatch = flag.String("shell-batch", "", "Execute shell command(s) from file and exit") +var withDisk = flag.String("with-disk", "", "Perform disk operation (-file-extract,-file-put,-file-delete)") +var fileExtract = flag.String("file-extract", "", "File to delete from disk (-with-disk)") +var filePut = flag.String("file-put", "", "File to put on disk (-with-disk)") +var fileDelete = flag.String("file-delete", "", "File to delete (-with-disk)") +var fileMkdir = flag.String("dir-create", "", "Directory to create (-with-disk)") +var fileCatalog = flag.Bool("catalog", false, "List disk contents (-with-disk)") +var quarantine = flag.Bool("quarantine", false, "Run -as-dupes and -whole-disk in quarantine mode") + +func main() { + + runtime.GOMAXPROCS(8) + + banner() + + //l.Default.Level = l.LevelCrit + + flag.Parse() + + var filterpath []string + + if *filterPath || *shell { + for _, v := range flag.Args() { + filterpath = append(filterpath, filepath.Clean(v)) + } + } + + //l.SILENT = !*logToFile + loggy.ECHO = *verbose + + if *withDisk != "" { + dsk, err := disk.NewDSKWrapper(defNibbler, *withDisk) + if err != nil { + os.Stderr.WriteString(err.Error()) + os.Exit(2) + } + commandVolumes[0] = dsk + commandTarget = 0 + switch { + case *fileExtract != "": + shellProcess("extract " + *fileExtract) + case *filePut != "": + shellProcess("put " + *filePut) + case *fileMkdir != "": + shellProcess("mkdir " + *fileMkdir) + case *fileDelete != "": + shellProcess("delete " + *fileDelete) + case *fileCatalog: + shellProcess("cat ") + default: + os.Stderr.WriteString("Additional flag required") + os.Exit(3) + } + + time.Sleep(5 * time.Second) + + os.Exit(0) + } + + // if *preCache { + // x := GetAllFiles("*_*_*_*.fgp") + // fmt.Println(len(x)) + // } + if *shellBatch != "" { + var data []byte + var err error + if *shellBatch == "stdin" { + data, err = ioutil.ReadAll(os.Stdin) + if err != nil { + os.Stderr.WriteString("Failed to read commands from stdin. Aborting") + os.Exit(1) + } + } else { + data, err = ioutil.ReadFile(*shellBatch) + if err != nil { + os.Stderr.WriteString("Failed to read commands from file. Aborting") + os.Exit(1) + } + } + lines := strings.Split(string(data), "\n") + for i, l := range lines { + r := shellProcess(l) + if r == -1 { + os.Stderr.WriteString(fmt.Sprintf("Script failed at line %d: %s\n", i+1, l)) + os.Exit(2) + } + if r == 999 { + os.Stderr.WriteString("Script terminated") + return + } + } + return + } + + if *shell { + var dsk *disk.DSKWrapper + var err error + if len(filterpath) > 0 { + fmt.Printf("Trying to load %s\n", filterpath[0]) + dsk, err = disk.NewDSKWrapper(defNibbler, filterpath[0]) + if err != nil { + fmt.Println("Error: " + err.Error()) + os.Exit(1) + } + } + shellDo(dsk) + os.Exit(0) + } + + defer func() { + + if fileExtractCounter > 0 { + os.Stderr.WriteString(fmt.Sprintf("%d files were extracted\n", fileExtractCounter)) + } + + }() + + if *searchFilename != "" { + searchForFilename(*searchFilename, filterpath) + return + } + + if *searchSHA != "" { + searchForSHA256(*searchSHA, filterpath) + return + } + + if *searchTEXT != "" { + searchForTEXT(*searchTEXT, filterpath) + return + } + + if *dir { + directory(filterpath, *dirFormat) + return + } + + if *allFileSubset { + allFilesSubsetReport(filterpath) + os.Exit(0) + } + + if *activeSectorSubset { + activeSectorsSubsetReport(filterpath) + os.Exit(0) + } + + if *allSectorSubset { + allSectorsSubsetReport(filterpath) + os.Exit(0) + } + + if *catDupes { + allFilesPartialReport(1.0, filterpath, "DUPLICATE CATALOG REPORT") + os.Exit(0) + } + + if *allFilePartial { + if *minSame == 0 && *maxDiff == 0 { + allFilesPartialReport(*similarity, filterpath, "") + } else if *minSame > 0 { + allFilesCustomReport(keeperAtLeastNSame, filterpath, fmt.Sprintf("AT LEAST %d FILES MATCH", *minSame)) + } else if *maxDiff > 0 { + allFilesCustomReport(keeperMaximumNDiff, filterpath, fmt.Sprintf("NO MORE THAN %d FILES DIFFER", *maxDiff)) + } + os.Exit(0) + } + + if *allSectorPartial { + allSectorsPartialReport(*similarity, filterpath) + os.Exit(0) + } + + if *activeSectorPartial { + activeSectorsPartialReport(*similarity, filterpath) + os.Exit(0) + } + + if *fileDupes { + fileDupeReport(filterpath) + os.Exit(0) + } + + if *wholeDupes { + if *quarantine { + quarantineWholeDisks(filterpath) + } else { + wholeDupeReport(filterpath) + } + os.Exit(0) + } + + if *activeDupes { + if *quarantine { + quarantineActiveDisks(filterpath) + } else { + activeDupeReport(filterpath) + } + os.Exit(0) + } + + _, e := os.Stat(*baseName) + if e != nil { + loggy.Get(0).Logf("Creating path %s", *baseName) + os.MkdirAll(*baseName, 0755) + } + + if *dskName == "" && *dskInfo == "" { + + var dsk *disk.DSKWrapper + var err error + if len(filterpath) > 0 { + fmt.Printf("Trying to load %s\n", filterpath[0]) + dsk, err = disk.NewDSKWrapper(defNibbler, filterpath[0]) + if err != nil { + fmt.Println("Error: " + err.Error()) + os.Exit(1) + } + } + shellDo(dsk) + os.Exit(0) + + } + + info, err := os.Stat(*dskName) + if err != nil { + loggy.Get(0).Errorf("Error stating file: %s", err.Error()) + os.Exit(2) + } + if info.IsDir() { + walk(*dskName) + } else { + indisk = make(map[disk.DiskFormat]int) + outdisk = make(map[disk.DiskFormat]int) + + panic.Do( + func() { + dsk, e := analyze(0, *dskName) + // handle any disk specific + if e == nil && *asPartial { + asPartialReport(dsk, *similarity, *reportFile, filterpath) + } else if e == nil && *filePartial { + filePartialReport(dsk, *similarity, *reportFile, filterpath) + } else if e == nil && *fileMatch != "" { + fileMatchReport(dsk, *fileMatch, filterpath) + } else if e == nil && *dir { + info := dsk.GetDirectory(*dirFormat) + fmt.Printf("Directory of %s:\n\n", dsk.Filename) + fmt.Println(info) + } + }, + func(r interface{}) { + loggy.Get(0).Errorf("Error processing volume: %s", *dskName) + loggy.Get(0).Errorf(string(debug.Stack())) + }, + ) + } +} diff --git a/make.sh b/make.sh new file mode 100755 index 0000000..368fb70 --- /dev/null +++ b/make.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +ARCHES="darwin-amd64 windows-386 windows-amd64 linux-386 linux-amd64 linux-arm freebsd-arm freebsd-amd64 freebsd-386" +PUBLISH="publish" + +mkdir -p "$PUBLISH" + +go get github.com/chzyer/readline + +exitState=0 +for arch in `echo $ARCHES`; do + export GOOS=`echo $arch | awk -F"-" '{print $1}'` + export GOARCH=`echo $arch | awk -F"-" '{print $2}'` + EXENAME="diskm8" + ZIPNAME="$PUBLISH/diskm8-$GOOS-$GOARCH.zip" + if [ "$GOOS" == "windows" ]; then + EXENAME="$EXENAME.exe" + fi + echo "Building $EXENAME..." + go build -o "$EXENAME" . + if [ "$?" == "0" ]; then + echo "Zipping -> $ZIPNAME" + zip "$ZIPNAME" "$EXENAME" "LICENSE" "README.md" "USAGE.md" + else + exit 2 + fi +done diff --git a/nibbler.go b/nibbler.go new file mode 100644 index 0000000..9826047 --- /dev/null +++ b/nibbler.go @@ -0,0 +1,13 @@ +package main + +type defaultNibbler struct{} + +var defNibbler = &defaultNibbler{} + +func (d *defaultNibbler) SetNibble(index int, value byte) { + +} + +func (d *defaultNibbler) GetNibble(index int) byte { + return 0 +} diff --git a/panic/panicwrap.go b/panic/panicwrap.go new file mode 100644 index 0000000..31a3487 --- /dev/null +++ b/panic/panicwrap.go @@ -0,0 +1,17 @@ +package panic + +func Do( f func(), h func(r interface{}) ) { + + defer func() { + + if r := recover(); r != nil { + h(r) + } + + }() + + f() + +} + + diff --git a/report.go b/report.go new file mode 100644 index 0000000..93318bb --- /dev/null +++ b/report.go @@ -0,0 +1,703 @@ +package main + +import ( + "fmt" + "os" + "sort" +) + +type DuplicateSource struct { + Fullpath string + Filename string + GSHA string + fingerprint string +} + +type DuplicateFileCollection struct { + data map[string][]DuplicateSource +} + +type DuplicateWholeDiskCollection struct { + data map[string][]DuplicateSource +} + +type DuplicateActiveSectorDiskCollection struct { + data map[string][]DuplicateSource + data_as map[string][]DuplicateSource +} + +func (dfc *DuplicateFileCollection) Add(checksum string, fullpath string, filename string, fgp string) { + + if dfc.data == nil { + dfc.data = make(map[string][]DuplicateSource) + } + + list, ok := dfc.data[checksum] + if !ok { + list = make([]DuplicateSource, 0) + } + + list = append(list, DuplicateSource{Fullpath: fullpath, Filename: filename, fingerprint: fgp}) + + dfc.data[checksum] = list + +} + +func (dfc *DuplicateWholeDiskCollection) Add(checksum string, fullpath string, fgp string) { + + if dfc.data == nil { + dfc.data = make(map[string][]DuplicateSource) + } + + list, ok := dfc.data[checksum] + if !ok { + list = make([]DuplicateSource, 0) + } + + list = append(list, DuplicateSource{Fullpath: fullpath, fingerprint: fgp}) + + dfc.data[checksum] = list + +} + +func (dfc *DuplicateActiveSectorDiskCollection) Add(checksum string, achecksum string, fullpath string, fgp string) { + + if dfc.data == nil { + dfc.data = make(map[string][]DuplicateSource) + } + + list, ok := dfc.data[achecksum] + if !ok { + list = make([]DuplicateSource, 0) + } + + list = append(list, DuplicateSource{Fullpath: fullpath, GSHA: checksum, fingerprint: fgp}) + + dfc.data[achecksum] = list + +} + +func (dfc *DuplicateFileCollection) Report(filename string) { + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + for sha256, list := range dfc.data { + + if len(list) > 1 { + + w.WriteString(fmt.Sprintf("\nChecksum %s duplicated %d times:\n", sha256, len(list))) + for i, v := range list { + w.WriteString(fmt.Sprintf(" %d) %s >> %s\n", i, v.Fullpath, v.Filename)) + } + + } + + } + +} + +func AggregateDuplicateFiles(d *Disk, collection interface{}) { + + for _, f := range d.Files { + + collection.(*DuplicateFileCollection).Add(f.SHA256, d.FullPath, f.Filename, d.source) + + } + +} + +func AggregateDuplicateWholeDisks(d *Disk, collection interface{}) { + + collection.(*DuplicateWholeDiskCollection).Add(d.SHA256, d.FullPath, d.source) + +} + +func AggregateDuplicateActiveSectorDisks(d *Disk, collection interface{}) { + + collection.(*DuplicateActiveSectorDiskCollection).Add(d.SHA256, d.SHA256Active, d.FullPath, d.source) + +} + +func (dfc *DuplicateWholeDiskCollection) Report(filename string) { + + var disksWithDupes int + var extras int + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + for sha256, list := range dfc.data { + + if len(list) > 1 { + + disksWithDupes++ + + original := list[0] + dupes := list[1:] + + w.WriteString("\n") + w.WriteString(fmt.Sprintf("Volume %s has %d duplicate(s):\n", original.Fullpath, len(dupes))) + for _, v := range dupes { + w.WriteString(fmt.Sprintf(" %s (sha256: %s)\n", v.Fullpath, sha256)) + extras++ + } + + } + + } + + w.WriteString("\n") + w.WriteString("SUMMARY\n") + w.WriteString("=======\n") + w.WriteString(fmt.Sprintf("Total disks which have duplicates: %d\n", disksWithDupes)) + w.WriteString(fmt.Sprintf("Total redundant copies found : %d\n", extras)) + +} + +func (dfc *DuplicateActiveSectorDiskCollection) Report(filename string) { + + var disksWithDupes int + var extras int + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + for sha256, list := range dfc.data { + + if len(list) > 1 { + + m := make(map[string]int) + for _, v := range list { + m[v.GSHA] = 1 + } + + if len(m) == 1 { + continue + } + + disksWithDupes++ + + original := list[0] + dupes := list[1:] + + w.WriteString("\n") + w.WriteString("--------------------------------------\n") + w.WriteString(fmt.Sprintf("Volume : %s\n", original.Fullpath)) + w.WriteString(fmt.Sprintf("Active SHA256: %s\n", sha256)) + w.WriteString(fmt.Sprintf("Global SHA256: %s\n", original.GSHA)) + w.WriteString(fmt.Sprintf("# Duplicates : %d\n", len(dupes))) + for i, v := range dupes { + w.WriteString("\n") + w.WriteString(fmt.Sprintf(" Duplicate #%d\n", i+1)) + w.WriteString(fmt.Sprintf(" = Volume : %s\n", v.Fullpath)) + w.WriteString(fmt.Sprintf(" = Active SHA256: %s\n", sha256)) + w.WriteString(fmt.Sprintf(" = Global SHA256: %s\n", v.GSHA)) + extras++ + } + w.WriteString("\n") + + } + + } + + w.WriteString("\n") + w.WriteString("SUMMARY\n") + w.WriteString("=======\n") + w.WriteString(fmt.Sprintf("Total disks which have duplicates: %d\n", disksWithDupes)) + w.WriteString(fmt.Sprintf("Total redundant copies found : %d\n", extras)) + +} + +func asPartialReport(d *Disk, t float64, filename string, pathfilter []string) { + matches := d.GetPartialMatchesWithThreshold(t, pathfilter) + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + w.WriteString(fmt.Sprintf("PARTIAL ACTIVE SECTOR MATCH REPORT FOR %s (Above %.2f%%)\n\n", d.Filename, 100*t)) + + //sort.Sort(ByMatchFactor(matches)) + sort.Sort(ByMatchFactor(matches)) + + w.WriteString(fmt.Sprintf("%d matches found\n\n", len(matches))) + for i := len(matches) - 1; i >= 0; i-- { + v := matches[i] + + w.WriteString(fmt.Sprintf("%.2f%%\t%s\n", v.MatchFactor*100, v.FullPath)) + + } + + w.WriteString("") +} + +func filePartialReport(d *Disk, t float64, filename string, pathfilter []string) { + matches := d.GetPartialFileMatchesWithThreshold(t, pathfilter) + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + w.WriteString(fmt.Sprintf("PARTIAL FILE MATCH REPORT FOR %s (Above %.2f%%)\n\n", d.Filename, 100*t)) + + //sort.Sort(ByMatchFactor(matches)) + sort.Sort(ByMatchFactor(matches)) + + w.WriteString(fmt.Sprintf("%d matches found\n\n", len(matches))) + for i := len(matches) - 1; i >= 0; i-- { + v := matches[i] + + w.WriteString(fmt.Sprintf("%.2f%%\t%s (%d missing, %d extras)\n", v.MatchFactor*100, v.FullPath, len(v.MissingFiles), len(v.ExtraFiles))) + for f1, f2 := range v.MatchFiles { + w.WriteString(fmt.Sprintf("\t == %s -> %s\n", f1.Filename, f2.Filename)) + } + for _, f := range v.MissingFiles { + w.WriteString(fmt.Sprintf("\t -- %s\n", f.Filename)) + } + for _, f := range v.ExtraFiles { + w.WriteString(fmt.Sprintf("\t ++ %s\n", f.Filename)) + } + w.WriteString("") + + } + + w.WriteString("") +} + +func fileMatchReport(d *Disk, filename string, pathfilter []string) { + + matches := d.GetFileMatches(filename, pathfilter) + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stdout + } + + w.WriteString(fmt.Sprintf("PARTIAL FILE MATCH REPORT FOR %s (File: %s)\n\n", d.Filename, filename)) + + w.WriteString(fmt.Sprintf("%d matches found\n\n", len(matches))) + for i, v := range matches { + + w.WriteString(fmt.Sprintf("%d)\t%s\n", i, v.FullPath)) + for f1, f2 := range v.MatchFiles { + w.WriteString(fmt.Sprintf("\t == %s -> %s\n", f1.Filename, f2.Filename)) + } + w.WriteString("") + + } + + w.WriteString("") +} + +func fileDupeReport(filter []string) { + + dfc := &DuplicateFileCollection{} + Aggregate(AggregateDuplicateFiles, dfc, filter) + + fmt.Println("DUPLICATE FILE REPORT") + fmt.Println() + + dfc.Report(*reportFile) + +} + +func wholeDupeReport(filter []string) { + + dfc := &DuplicateWholeDiskCollection{} + Aggregate(AggregateDuplicateWholeDisks, dfc, filter) + + fmt.Println("DUPLICATE WHOLE DISK REPORT") + fmt.Println() + + dfc.Report(*reportFile) + +} + +func activeDupeReport(filter []string) { + + dfc := &DuplicateActiveSectorDiskCollection{} + Aggregate(AggregateDuplicateActiveSectorDisks, dfc, filter) + + fmt.Println("DUPLICATE ACTIVE SECTORS DISK REPORT") + fmt.Println() + + dfc.Report(*reportFile) + +} + +func allFilesPartialReport(t float64, filter []string, oheading string) { + + matches := CollectFilesOverlapsAboveThreshold(t, filter) + + if *csvOut { + dumpFileOverlapCSV(matches, *reportFile) + return + } + + if oheading != "" { + fmt.Println(oheading + "\n") + } else { + fmt.Printf("PARTIAL ALL FILE MATCH REPORT (Above %.2f%%)\n\n", 100*t) + } + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, ratio := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: %.2f%% Match to %s\n", 100*ratio, k) + for f1, f2 := range matchdata.files[k] { + fmt.Printf(" == %s -> %s\n", f1.Filename, f2.Filename) + } + for _, f := range matchdata.missing[k] { + fmt.Printf(" -- %s\n", f.Filename) + } + for _, f := range matchdata.extras[k] { + fmt.Printf(" ++ %s\n", f.Filename) + } + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func allSectorsPartialReport(t float64, filter []string) { + + matches := CollectSectorOverlapsAboveThreshold(t, filter, GetAllDiskSectors) + + if *csvOut { + dumpSectorOverlapCSV(matches, *reportFile) + return + } + + fmt.Printf("NON-ZERO SECTOR MATCH REPORT (Above %.2f%%)\n\n", 100*t) + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, ratio := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: %.2f%% Match to %s\n", 100*ratio, k) + fmt.Printf(" == %d Sectors matched\n", len(matchdata.same[k])) + fmt.Printf(" -- %d Sectors missing\n", len(matchdata.missing[k])) + fmt.Printf(" ++ %d Sectors extra\n", len(matchdata.extras[k])) + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func activeSectorsPartialReport(t float64, filter []string) { + + matches := CollectSectorOverlapsAboveThreshold(t, filter, GetActiveDiskSectors) + + if *csvOut { + dumpSectorOverlapCSV(matches, *reportFile) + return + } + + fmt.Printf("PARTIAL ACTIVE SECTOR MATCH REPORT (Above %.2f%%)\n\n", 100*t) + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, ratio := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: %.2f%% Match to %s\n", 100*ratio, k) + fmt.Printf(" == %d Sectors matched\n", len(matchdata.same[k])) + fmt.Printf(" -- %d Sectors missing\n", len(matchdata.missing[k])) + fmt.Printf(" ++ %d Sectors extra\n", len(matchdata.extras[k])) + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func allFilesSubsetReport(filter []string) { + + matches := CollectFileSubsets(filter) + + if *csvOut { + dumpFileOverlapCSV(matches, *reportFile) + return + } + + fmt.Printf("SUBSET DISK FILE MATCH REPORT\n\n") + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, _ := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: Is a file subset of %s\n", k) + for f1, f2 := range matchdata.files[k] { + fmt.Printf(" == %s -> %s\n", f1.Filename, f2.Filename) + } + for _, f := range matchdata.missing[k] { + fmt.Printf(" -- %s\n", f.Filename) + } + for _, f := range matchdata.extras[k] { + fmt.Printf(" ++ %s\n", f.Filename) + } + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func activeSectorsSubsetReport(filter []string) { + + matches := CollectSectorSubsets(filter, GetActiveDiskSectors) + + if *csvOut { + dumpSectorOverlapCSV(matches, *reportFile) + return + } + + fmt.Printf("ACTIVE SECTOR SUBSET MATCH REPORT\n\n") + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, _ := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: Is a subset (based on active sectors) of %s\n", k) + fmt.Printf(" == %d Sectors matched\n", len(matchdata.same[k])) + fmt.Printf(" ++ %d Sectors extra\n", len(matchdata.extras[k])) + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func allSectorsSubsetReport(filter []string) { + + matches := CollectSectorSubsets(filter, GetAllDiskSectors) + + if *csvOut { + dumpSectorOverlapCSV(matches, *reportFile) + return + } + + fmt.Printf("NON-ZERO SECTOR SUBSET MATCH REPORT\n\n") + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, _ := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: Is a subset (based on active sectors) of %s\n", k) + fmt.Printf(" == %d Sectors matched\n", len(matchdata.same[k])) + fmt.Printf(" ++ %d Sectors extra\n", len(matchdata.extras[k])) + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} + +func dumpFileOverlapCSV(matches map[string]*FileOverlapRecord, filename string) { + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stderr + } + + w.WriteString("MATCH,DISK1,FILENAME1,DISK2,FILENAME2,EXISTS\n") + for disk1, matchdata := range matches { + for disk2, match := range matchdata.percent { + for f1, f2 := range matchdata.files[disk2] { + w.WriteString(fmt.Sprintf(`%.2f,"%s","%s","%s","%s",%s`, match, disk1, f1.Filename, disk2, f2.Filename, "Y") + "\n") + } + for _, f1 := range matchdata.missing[disk2] { + w.WriteString(fmt.Sprintf(`%.2f,"%s","%s","%s","%s",%s`, match, disk1, f1.Filename, disk2, "", "N") + "\n") + } + for _, f2 := range matchdata.extras[disk2] { + w.WriteString(fmt.Sprintf(`%.2f,"%s","%s","%s","%s",%s`, match, disk1, "", disk2, f2.Filename, "N") + "\n") + } + } + } + + if filename != "" { + fmt.Println("\nWrote " + filename + "\n") + } + +} + +func dumpSectorOverlapCSV(matches map[string]*SectorOverlapRecord, filename string) { + + var w *os.File + var err error + + if filename != "" { + w, err = os.Create(filename) + if err != nil { + return + } + defer w.Close() + } else { + w = os.Stderr + } + + w.WriteString("MATCH,DISK1,DISK2,SAME,MISSING,EXTRA\n") + for disk1, matchdata := range matches { + for disk2, match := range matchdata.percent { + w.WriteString(fmt.Sprintf(`%.2f,"%s","%s",%d,%d,%d`, match, disk1, disk2, len(matchdata.same[disk2]), len(matchdata.missing[disk2]), len(matchdata.extras[disk2])) + "\n") + } + } + + if filename != "" { + fmt.Println("\nWrote " + filename + "\n") + } + +} + +func keeperAtLeastNSame(d1, d2 string, v *FileOverlapRecord) bool { + + return len(v.files[d2]) >= *minSame + +} + +func keeperMaximumNDiff(d1, d2 string, v *FileOverlapRecord) bool { + + return len(v.files[d2]) > 0 && (len(v.missing[d2])+len(v.extras[d2])) <= *maxDiff + +} + +func allFilesCustomReport(keep func(d1, d2 string, v *FileOverlapRecord) bool, filter []string, oheading string) { + + matches := CollectFilesOverlapsCustom(keep, filter) + + if *csvOut { + dumpFileOverlapCSV(matches, *reportFile) + return + } + + fmt.Println(oheading + "\n") + + fmt.Printf("%d matches found\n\n", len(matches)) + for volumename, matchdata := range matches { + + fmt.Printf("Disk: %s\n", volumename) + + for k, ratio := range matchdata.percent { + fmt.Println() + fmt.Printf(" :: %.2f%% Match to %s\n", 100*ratio, k) + for f1, f2 := range matchdata.files[k] { + fmt.Printf(" == %s -> %s\n", f1.Filename, f2.Filename) + } + for _, f := range matchdata.missing[k] { + fmt.Printf(" -- %s\n", f.Filename) + } + for _, f := range matchdata.extras[k] { + fmt.Printf(" ++ %s\n", f.Filename) + } + fmt.Println() + } + + fmt.Println() + + } + + fmt.Println() +} diff --git a/search.go b/search.go new file mode 100644 index 0000000..1624a0b --- /dev/null +++ b/search.go @@ -0,0 +1,206 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type SearchResultContext int + +const ( + SRC_UNKNOWN SearchResultContext = iota + SRC_FILE + SRC_DISK +) + +type SearchResultItem struct { + DiskPath string + File *DiskFile +} + +func searchForFilename(filename string, filter []string) { + + fd := GetAllFiles("*_*_*_*.fgp", filter) + + fmt.Printf("Filter: %s\n", filter) + + fmt.Println() + fmt.Println() + + fmt.Printf("SEARCH RESULTS FOR '%s'\n", filename) + + fmt.Println() + + for diskname, list := range fd { + //fmt.Printf("Checking: %s\n", diskname) + for _, f := range list { + if strings.Contains(strings.ToLower(f.Filename), strings.ToLower(filename)) { + fmt.Printf("%32s:\n %s (%s, %d bytes, sha: %s)\n\n", diskname, f.Filename, f.Type, f.Size, f.SHA256) + if *extract == "@" { + ExtractFile(diskname, f, *adornedCP, false) + } else if *extract == "#" { + ExtractDisk(diskname) + } + } + } + } + +} + +func searchForSHA256(sha string, filter []string) { + + fd := GetAllFiles("*_*_*_*.fgp", filter) + + fmt.Println() + fmt.Println() + + fmt.Printf("SEARCH RESULTS FOR SHA256 '%s'\n", sha) + + fmt.Println() + + for diskname, list := range fd { + for _, f := range list { + if f.SHA256 == sha { + fmt.Printf("%32s:\n %s (%s, %d bytes, sha: %s)\n\n", diskname, f.Filename, f.Type, f.Size, f.SHA256) + if *extract == "@" { + ExtractFile(diskname, f, *adornedCP, false) + } else if *extract == "#" { + ExtractDisk(diskname) + } + } + } + } + +} + +func searchForTEXT(text string, filter []string) { + + fd := GetAllFiles("*_*_*_*.fgp", filter) + + fmt.Println() + fmt.Println() + + fmt.Printf("SEARCH RESULTS FOR TEXT CONTENT '%s'\n", text) + + fmt.Println() + + for diskname, list := range fd { + for _, f := range list { + if strings.Contains(strings.ToLower(string(f.Text)), strings.ToLower(text)) { + fmt.Printf("%32s:\n %s (%s, %d bytes, sha: %s)\n\n", diskname, f.Filename, f.Type, f.Size, f.SHA256) + if *extract == "@" { + ExtractFile(diskname, f, *adornedCP, false) + } else if *extract == "#" { + ExtractDisk(diskname) + } + } + } + } + +} + +func directory(filter []string, format string) { + + fd := GetAllFiles("*_*_*_*.fgp", filter) + + fmt.Println() + fmt.Println() + + fmt.Println() + + for diskname, list := range fd { + fmt.Printf("CATALOG RESULTS FOR '%s'\n", diskname) + //fmt.Printf("Checking: %s\n", diskname) + out := "" + for _, file := range list { + tmp := format + // size + tmp = strings.Replace(tmp, "{size:blocks}", fmt.Sprintf("%3d Blocks", file.Size/256+1), -1) + tmp = strings.Replace(tmp, "{size:kb}", fmt.Sprintf("%4d Kb", file.Size/1024+1), -1) + tmp = strings.Replace(tmp, "{size:b}", fmt.Sprintf("%6d Bytes", file.Size), -1) + tmp = strings.Replace(tmp, "{size}", fmt.Sprintf("%6d", file.Size), -1) + // format + tmp = strings.Replace(tmp, "{filename}", fmt.Sprintf("%-36s", file.Filename), -1) + // type + tmp = strings.Replace(tmp, "{type}", fmt.Sprintf("%-20s", file.Type), -1) + // sha256 + tmp = strings.Replace(tmp, "{sha256}", file.SHA256, -1) + + out += tmp + "\n" + + if *extract == "@" { + ExtractFile(diskname, file, *adornedCP, false) + } else if *extract == "#" { + ExtractDisk(diskname) + } + } + fmt.Println(out + "\n\n") + } + +} + +var fileExtractCounter int + +func ExtractFile(diskname string, fd *DiskFile, adorned bool, local bool) error { + + var name string + + if adorned { + name = fd.GetNameAdorned() + } else { + name = fd.GetName() + } + + path := binpath() + "/extract" + diskname + + if local { + ext := filepath.Ext(diskname) + base := strings.Replace(filepath.Base(diskname), ext, "", -1) + path = "./" + base + } + + if path != "." { + os.MkdirAll(path, 0755) + } + + //fmt.Printf("FD.EXT=%s\n", fd.Ext) + + f, err := os.Create(path + "/" + name) + if err != nil { + return err + } + defer f.Close() + f.Write(fd.Data) + os.Stderr.WriteString("Extracted file to " + path + "/" + name + "\n") + + if strings.ToLower(fd.Ext) == "int" || strings.ToLower(fd.Ext) == "bas" || strings.ToLower(fd.Ext) == "txt" { + f, err := os.Create(path + "/" + name + ".ASC") + if err != nil { + return err + } + defer f.Close() + f.Write(fd.Text) + os.Stderr.WriteString("Extracted file to " + path + "/" + name + ".ASC\n") + } + + //os.Stderr.WriteString("Extracted file to " + path + "/" + name) + + fileExtractCounter++ + + return nil + +} + +func ExtractDisk(diskname string) error { + path := binpath() + "/extract" + diskname + os.MkdirAll(path, 0755) + data, err := ioutil.ReadFile(diskname) + if err != nil { + return err + } + target := path + "/" + filepath.Base(diskname) + return ioutil.WriteFile(target, data, 0755) +} diff --git a/shell.go b/shell.go new file mode 100644 index 0000000..1e39709 --- /dev/null +++ b/shell.go @@ -0,0 +1,1732 @@ +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "runtime/debug" + "strings" + "time" + + "path/filepath" + + "sort" + + "os" + + "regexp" + + "strconv" + + "errors" + + "github.com/chzyer/readline" + "github.com/paleotronic/diskm8/disk" + "github.com/paleotronic/diskm8/loggy" + "github.com/paleotronic/diskm8/panic" +) + +const MAXVOL = 8 + +var commandList map[string]*shellCommand +var commandVolumes [MAXVOL]*disk.DSKWrapper +var commandTarget int = -1 +var commandPath string + +func mountDsk(dsk *disk.DSKWrapper) (int, error) { + + var fr []int + + for i, d := range commandVolumes { + if d == nil { + fr = append(fr, i) + } else if dsk.Filename == d.Filename { + return i, nil + } + } + + if len(fr) == 0 { + return -1, errors.New("No free slots") + } + + commandVolumes[fr[0]] = dsk + + return fr[0], nil + +} + +func smartSplit(line string) (string, []string) { + + var out []string + + var inqq bool + var lastEscape bool + var chunk string + + add := func() { + if chunk != "" { + out = append(out, chunk) + chunk = "" + } + } + + for _, ch := range line { + switch { + case ch == '"': + inqq = !inqq + add() + case ch == ' ': + if inqq || lastEscape { + chunk += string(ch) + } else { + add() + } + lastEscape = false + case ch == '\\' && !inqq: + lastEscape = true + default: + chunk += string(ch) + } + } + + add() + + if len(out) == 0 { + return "", out + } + + return out[0], out[1:] +} + +func getPrompt(wp string, t int) string { + + if t == -1 || commandVolumes[t] == nil { + return fmt.Sprintf("dsk:%d:%s:%s> ", 0, "", wp) + } + + dsk := commandVolumes[t] + + if dsk != nil { + return fmt.Sprintf("dsk:%d:%s:%s> ", t, filepath.Base(dsk.Filename), wp) + } + return "dsk> " +} + +type shellCommand struct { + Name string + Description string + MinArgs, MaxArgs int + Code func(args []string) int + NeedsMount bool + Context shellCommandContext + Text []string +} + +type shellCommandContext int + +const ( + sccNone shellCommandContext = 1 << iota + sccLocal + sccDiskFile + sccCommand + sccReportName + sccAnyFile = sccDiskFile | sccLocal + sccAny = sccAnyFile | sccCommand +) + +type shellCompleter struct { +} + +func hasPrefix(str []rune, prefix []rune) bool { + if len(prefix) > len(str) { + return false + } + for i := 0; i < len(prefix); i++ { + if str[i] != prefix[i] { + return false + } + } + return true +} + +func (sc *shellCompleter) Do(line []rune, pos int) ([][]rune, int) { + + prefix := "" + chunk := "" + for _, ch := range line { + if ch == ' ' { + prefix = chunk + break + } else { + chunk += string(ch) + } + } + + chunk = "" + cprefix := "" + var lastEscape bool + for i := 0; i < pos; i++ { + ch := line[i] + switch { + case ch == '\\': + lastEscape = true + case ch == ' ' && !lastEscape: + cprefix = chunk + chunk = "" + lastEscape = false + default: + chunk += string(ch) + } + } + if chunk != "" { + cprefix = chunk + } + + var context shellCommandContext = sccNone + cmd, match := commandList[prefix] + if match { + context = cmd.Context + } else { + context = sccCommand + } + + var items [][]rune + switch context { + case sccCommand: + for k, _ := range commandList { + items = append(items, []rune(k)) + } + case sccDiskFile: + if commandTarget == -1 || commandVolumes[commandTarget] == nil { + return [][]rune(nil), 0 + } + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + info, err := analyze(0, fullpath) + if err != nil { + return [][]rune(nil), 0 + } + for _, f := range info.Files { + items = append(items, []rune(f.Filename)) + } + case sccLocal: + files, err := filepath.Glob(cprefix + "*") + //fmt.Println(err) + if err != nil { + return items, 0 + } + for _, v := range files { + items = append(items, []rune(v)) + //fmt.Println("thing:", v) + } + } + + if len(items) == 0 { + return [][]rune(nil), 0 + } + + //fmt.Printf("Context = %d, CPrefix=%s, Items=%v\n", context, cprefix, items) + + var filt [][]rune + for _, v := range items { + if hasPrefix(v, []rune(cprefix)) { + filt = append(filt, shellEscape(v[len(cprefix):])) + } + } + return filt, len(cprefix) +} + +func shellEscape(str []rune) []rune { + out := make([]rune, 0) + for _, v := range str { + if v == ' ' { + out = append(out, '\\') + } + out = append(out, v) + } + return out +} + +func init() { + commandList = map[string]*shellCommand{ + "mount": &shellCommand{ + Name: "mount", + Description: "Mount a disk image", + MinArgs: 1, + MaxArgs: 1, + Code: shellMount, + NeedsMount: false, + Context: sccLocal, + Text: []string{ + "mount ", + "", + "Mounts disk and switches to the new slot", + }, + }, + "unmount": &shellCommand{ + Name: "unmount", + Description: "unmount disk image", + MinArgs: 0, + MaxArgs: 1, + Code: shellUnmount, + NeedsMount: true, + Context: sccLocal, + Text: []string{ + "unmount ", + "", + "Unmount the disk in the specified slot (or current slot)", + }, + }, + "extract": &shellCommand{ + Name: "extract", + Description: "extract file from disk image", + MinArgs: 1, + MaxArgs: -1, + Code: shellExtract, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "extract ", + "", + "Extracts files from current disk", + }, + }, + "help": &shellCommand{ + Name: "help", + Description: "Shows this help", + MinArgs: 0, + MaxArgs: 1, + Code: shellHelp, + NeedsMount: false, + Context: sccCommand, + Text: []string{ + "help ", + "", + "Display specific help for command or list of commands", + }, + }, + "info": &shellCommand{ + Name: "info", + Description: "Information about the current disk", + MinArgs: -1, + MaxArgs: -1, + Code: shellInfo, + NeedsMount: true, + Context: sccNone, + Text: []string{ + "info", + "", + "Display information on current disk", + }, + }, + "analyze": &shellCommand{ + Name: "analyze", + Description: "Process disk using diskm8 analytics", + MinArgs: -1, + MaxArgs: -1, + Code: shellAnalyze, + NeedsMount: true, + Context: sccNone, + Text: []string{ + "analyze", + "", + "Display detailed diskm8 information on current disk", + }, + }, + "quit": &shellCommand{ + Name: "quit", + Description: "Leave this place", + MinArgs: -1, + MaxArgs: -1, + Code: shellQuit, + NeedsMount: false, + Context: sccNone, + }, + "cat": &shellCommand{ + Name: "cat", + Description: "Display file information", + MinArgs: 0, + MaxArgs: 1, + Code: shellCat, + NeedsMount: true, + Context: sccNone, + Text: []string{ + "cat []", + "", + "List files on current disk (can use wildcards).", + }, + }, + "mkdir": &shellCommand{ + Name: "mkdir", + Description: "Create a directory on disk", + MinArgs: 1, + MaxArgs: 1, + Code: shellMkdir, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "mkdir ", + "", + "Create directory on current disk (if supported)", + }, + }, + "put": &shellCommand{ + Name: "put", + Description: "Copy local file to disk", + MinArgs: 1, + MaxArgs: 1, + Code: shellPut, + NeedsMount: true, + Context: sccLocal, + Text: []string{ + "put ", + "", + "Write local file to current disk", + }, + }, + "delete": &shellCommand{ + Name: "delete", + Description: "Remove file from disk", + MinArgs: 1, + MaxArgs: 1, + Code: shellDelete, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "delete ", + "", + "Delete file from current disk", + }, + }, + "ingest": &shellCommand{ + Name: "ingest", + Description: "Ingest directory containing disks (or single disk) into system", + MinArgs: 1, + MaxArgs: 1, + Code: shellIngest, + NeedsMount: false, + Context: sccLocal, + Text: []string{ + "ingest ", + "", + "Catalog diskfile into diskm8 database.", + }, + }, + "lock": &shellCommand{ + Name: "lock", + Description: "Lock file on the disk", + MinArgs: 1, + MaxArgs: 1, + Code: shellLock, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "lock ", + "", + "Make file on disk read-only", + }, + }, + "unlock": &shellCommand{ + Name: "unlock", + Description: "Unlock file on the disk", + MinArgs: 1, + MaxArgs: 1, + Code: shellUnlock, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "unlock ", + "", + "Make file on disk writable", + }, + }, + "ls": &shellCommand{ + Name: "ls", + Description: "List local files", + MinArgs: 0, + MaxArgs: 999, + Code: shellListFiles, + NeedsMount: false, + Context: sccLocal, + Text: []string{ + "ls ", + "", + "List local files", + }, + }, + "cd": &shellCommand{ + Name: "cd", + Description: "Change local path", + MinArgs: 0, + MaxArgs: 1, + Code: shellCd, + NeedsMount: false, + Context: sccLocal, + Text: []string{ + "cd ", + "", + "Change local directory", + }, + }, + "disks": &shellCommand{ + Name: "disks", + Description: "List mounted volumes", + MinArgs: 0, + MaxArgs: 0, + Code: shellDisks, + NeedsMount: false, + Context: sccNone, + Text: []string{ + "disks", + "", + "List all mounted volumes", + }, + }, + "target": &shellCommand{ + Name: "target", + Description: "Select mounted volume as default", + MinArgs: 1, + MaxArgs: 1, + Code: shellPrefix, + NeedsMount: false, + Context: sccNone, + Text: []string{ + "target ", + "", + "Select slot as default for commands", + }, + }, + "copy": &shellCommand{ + Name: "copy", + Description: "Copy files from one volume to another", + MinArgs: 2, + MaxArgs: 999, + Code: shellD2DCopy, + NeedsMount: false, + Context: sccDiskFile, + Text: []string{ + "copy [:] :[]", + "", + "Copy files from one mounted disk to another.", + "Example:", + "copy 0:*.system 1:", + }, + }, + "move": &shellCommand{ + Name: "move", + Description: "Move files from one volume to another", + MinArgs: 2, + MaxArgs: 999, + Code: shellD2DCopy, + NeedsMount: false, + Context: sccDiskFile, + Text: []string{ + "move [:] :[]", + "", + "Move files from one mounted disk to another.", + "Example:", + "move 0:*.system 1:", + }, + }, + "rename": &shellCommand{ + Name: "rename", + Description: "Rename a file on the disk", + MinArgs: 2, + MaxArgs: 2, + Code: shellRename, + NeedsMount: true, + Context: sccDiskFile, + Text: []string{ + "rename ", + "", + "Rename a file on a disk.", + }, + }, + "report": &shellCommand{ + Name: "report", + Description: "Run a report", + MinArgs: 1, + MaxArgs: 999, + Code: shellReport, + NeedsMount: false, + Context: sccDiskFile, + Text: []string{ + "report []", + "", + "Reports:", + "as-dupes Active sector dupes report (-as-dupes at command line)", + "file-dupes File dupes report (-file-dupes at command line)", + "whole-dupes Whole disk dupes report (-whole-dupes at command line)", + }, + }, + "search": &shellCommand{ + Name: "search", + Description: "Run a search", + MinArgs: 1, + MaxArgs: 999, + Code: shellSearch, + NeedsMount: false, + Context: sccDiskFile, + Text: []string{ + "search []", + "", + "Searches:", + "filename Search by filename", + "text Search for files containing tex", + "hash Search for files with hash", + }, + }, + "quarantine": &shellCommand{ + Name: "quarantine", + Description: "Like report, but allow moving dupes to a backup folder", + MinArgs: 1, + MaxArgs: 999, + Code: shellQuarantine, + NeedsMount: false, + Context: sccDiskFile, + Text: []string{ + "quarantine []", + "", + "Scans:", + "as-dupes Active sector dupes report (-as-dupes at command line)", + "file-dupes File dupes report (-file-dupes at command line)", + "whole-dupes Whole disk dupes report (-whole-dupes at command line)", + }, + }, + } +} + +func shellProcess(line string) int { + line = strings.TrimSpace(line) + + verb, args := smartSplit(line) + + if verb != "" { + verb = strings.ToLower(verb) + command, ok := commandList[verb] + if ok { + fmt.Println() + var cok = true + if command.MinArgs != -1 { + if len(args) < command.MinArgs { + os.Stderr.WriteString(fmt.Sprintf("%s expects at least %d arguments\n", verb, command.MinArgs)) + cok = false + } + } + if command.MaxArgs != -1 { + if len(args) > command.MaxArgs { + os.Stderr.WriteString(fmt.Sprintf("%s expects at most %d arguments\n", verb, command.MaxArgs)) + cok = false + } + } + if command.NeedsMount { + if commandTarget == -1 || commandVolumes[commandTarget] == nil { + os.Stderr.WriteString(fmt.Sprintf("%s only works on mounted disks\n", verb)) + cok = false + } + } + if cok { + r := command.Code(args) + fmt.Println() + return r + } else { + return -1 + } + } else { + os.Stderr.WriteString(fmt.Sprintf("Unrecognized command: %s\n", verb)) + return -1 + } + } + + return 0 +} + +func shellDo(dsk *disk.DSKWrapper) { + + //commandVolumes = dsk + commandPath := "" + + ac := &shellCompleter{} + + rl, err := readline.NewEx(&readline.Config{ + Prompt: getPrompt(commandPath, commandTarget), + HistoryFile: binpath() + "/.shell_history", + DisableAutoSaveHistory: false, + AutoComplete: ac, + }) + if err != nil { + //fmt.Println("Error rl:", err) + os.Exit(2) + } + defer rl.Close() + + running := true + + for running { + line, err := rl.Readline() + if err != nil { + //fmt.Println("Error:", err) + break + } + + r := shellProcess(line) + if r == 999 { + //fmt.Println("exit 999") + return + } + + rl.SetPrompt(getPrompt(commandPath, commandTarget)) + } + +} + +func shellMount(args []string) int { + if len(args) != 1 { + fmt.Println("mount expects a diskfile") + return -1 + } + + dsk, err := disk.NewDSKWrapper(defNibbler, args[0]) + if err != nil { + os.Stderr.WriteString("Error:" + err.Error() + "\n") + return -1 + } + + slotid, err := mountDsk(dsk) + if err != nil { + os.Stderr.WriteString("Error:" + err.Error() + "\n") + return -1 + } + + commandTarget = slotid + os.Stderr.WriteString(fmt.Sprintf("mount disk in slot %d\n", slotid)) + + return 0 +} + +func shellUnmount(args []string) int { + + if len(args) > 0 { + if shellPrefix(args) == -1 { + return -1 + } + } + + if commandVolumes[commandTarget] != nil { + + commandVolumes[commandTarget] = nil + commandPath = "" + + os.Stderr.WriteString("Unmounted volume\n") + + } + + return 0 +} + +func shellHelp(args []string) int { + + if len(args) == 0 { + keys := make([]string, 0) + for k, _ := range commandList { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + info := commandList[k] + fmt.Printf("%-10s %s\n", info.Name, info.Description) + } + } else { + command := strings.ToLower(args[0]) + if details, ok := commandList[command]; ok { + if details.Text != nil { + for _, l := range details.Text { + fmt.Println(l) + } + } else { + os.Stderr.WriteString("No help available for " + command) + } + } else { + os.Stderr.WriteString("No help available for " + command) + } + } + + return 0 +} + +func shellInfo(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + fmt.Printf("Disk path : %s\n", fullpath) + fmt.Printf("Disk type : %s\n", commandVolumes[commandTarget].Format.String()) + fmt.Printf("Sector Order: %s\n", commandVolumes[commandTarget].Layout.String()) + fmt.Printf("Size : %d bytes\n", len(commandVolumes[commandTarget].Data)) + + return 0 +} + +func shellQuit(args []string) int { + + return 999 + +} + +func shellCat(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + info, err := analyze(0, fullpath) + if err != nil { + return -1 + } + + bs := 256 + if info.FormatID.ID == disk.DF_PASCAL || info.FormatID.ID == disk.DF_PRODOS || + info.FormatID.ID == disk.DF_PRODOS_800KB || info.FormatID.ID == disk.DF_PRODOS_400KB || + info.FormatID.ID == disk.DF_PRODOS_CUSTOM { + bs = 512 + } + + pattern := "*" + if len(args) > 0 { + pattern = args[0] + } + + files, _ := globDisk(commandTarget, pattern) + + fmt.Printf("%-33s %6s %2s %-23s %s\n", "NAME", "BLOCKS", "RO", "KIND", "ADDITONAL") + for _, f := range files { + add := "" + locked := " " + if f.LoadAddress != 0 { + add = fmt.Sprintf("(A$%.4X)", f.LoadAddress) + } + if f.Locked { + locked = "Y" + } + fmt.Printf("%-33s %6d %2s %-23s %s\n", f.Filename, (f.Size/bs)+1, locked, f.Type, add) + } + + free := 0 + used := 0 + for _, v := range info.Bitmap { + if v { + used++ + } else { + free++ + } + } + + fmt.Printf("\nUSED: %-20d FREE: %-20d\n", used, free) + + return 0 + +} + +func shellCd(args []string) int { + + if len(args) > 0 { + err := os.Chdir(args[0]) + if err != nil { + os.Stderr.WriteString("Change directory failed: " + err.Error() + "\n") + return -1 + } + } + + wd, _ := os.Getwd() + os.Stderr.WriteString("Working directory is now " + wd + "\n") + return 0 + +} + +func shellListFiles(args []string) int { + + bs := 256 + + if len(args) == 0 { + wd, _ := os.Getwd() + args = append(args, wd+"/*.*") + } + + for _, a := range args { + + files, err := filepath.Glob(a) + if err != nil { + os.Stderr.WriteString("Error reading path " + a + ": " + err.Error() + "\n") + continue + } + + fmt.Printf("%6s %2s %-23s %s\n", "BLOCKS", "RO", "KIND", "NAME") + for _, f := range files { + locked := " " + fi, _ := os.Stat(f) + if fi.Mode().Perm()&0100 != 0100 { + locked = "Y" + } + fmt.Printf("%6d %2s %-23s %s\n", (int(fi.Size())/bs)+1, locked, "Local file", fi.Name()) + } + } + + return 0 +} + +func shellAnalyze(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + info, err := analyze(0, fullpath) + if err != nil { + return -1 + } + + fmt.Printf("Format: %s\n", info.FormatID) + fmt.Printf("Tracks: %d, Sectors: %d\n", info.Tracks, info.Sectors) + + return 0 +} + +func shellExtract(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + fmt.Println("Extract:", args[0]) + + files, _ := globDisk(commandTarget, args[0]) + + for _, f := range files { + + err := ExtractFile(fullpath, f, true, true) + if err == nil { + fmt.Println("OK") + } else { + fmt.Println("FAILED") + return -1 + } + + } + + return 0 + +} + +func formatIn(f disk.DiskFormatID, list []disk.DiskFormatID) bool { + for _, v := range list { + if v == f { + return true + } + } + return false +} + +func fts() string { + t := time.Now() + return fmt.Sprintf( + "%.4d%.2d%.2d%.2d%.2d%.2d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second(), + ) +} + +func backupFile(path string) error { + data, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + path = strings.Replace(path, ":", "", -1) + path = strings.Replace(path, "\\", "/", -1) + + bpath := binpath() + "/backup/" + path + "." + fts() + os.MkdirAll(filepath.Dir(bpath), 0755) + + f, err := os.Create(bpath) + if err != nil { + return err + } + f.Write(data) + f.Close() + + os.Stderr.WriteString("Backed up disk to: " + bpath + "\n") + + return nil +} + +func saveDisk(dsk *disk.DSKWrapper, path string) error { + + backupFile(path) + + f, e := os.Create(path) + if e != nil { + return e + } + defer f.Close() + f.Write(dsk.Data) + + fmt.Println("Updated disk " + path) + return nil +} + +func shellMkdir(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + path := "" + name := args[0] + if strings.Contains(name, "/") { + path = filepath.Dir(name) + name = filepath.Base(name) + } + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + e := commandVolumes[commandTarget].PRODOSCreateDirectory(path, name) + if e != nil { + fmt.Println(e) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + } else { + fmt.Println("Do not support Mkdir on " + commandVolumes[commandTarget].Format.String() + " currently.") + return 0 + } + + return 0 + +} + +func isASCII(in []byte) bool { + for _, v := range in { + if v > 128 { + return false + } + } + return true +} + +func shellPut(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + data, err := ioutil.ReadFile(args[0]) + if err != nil { + return -1 + } + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_DOS_SECTORS_13, disk.DF_DOS_SECTORS_16}) { + addr := int64(0x0801) + name := filepath.Base(args[0]) + kind := disk.FileTypeAPP + reSpecial := regexp.MustCompile("(?i)^(.+)[#](0x[a-fA-F0-9]+)[.]([A-Za-z]+)$") + ext := strings.Trim(filepath.Ext(name), ".") + if reSpecial.MatchString(name) { + m := reSpecial.FindAllStringSubmatch(name, -1) + name = m[0][1] + ext = strings.ToLower(m[0][3]) + addrStr := m[0][2] + addr, _ = strconv.ParseInt(addrStr, 0, 32) + } else { + name = strings.Replace(name, "."+ext, "", -1) + } + + kind = disk.AppleDOSFileTypeFromExt(ext) + + if strings.HasSuffix(args[0], ".INT.ASC") { + kind = disk.FileTypeINT + } else if strings.HasSuffix(args[0], ".APP.ASC") { + kind = disk.FileTypeAPP + } + + if kind == disk.FileTypeAPP && isASCII(data) { + lines := strings.Split(string(data), "\n") + data = disk.ApplesoftTokenize(lines) + } else if kind == disk.FileTypeINT && isASCII(data) { + lines := strings.Split(string(data), "\n") + data = disk.IntegerTokenize(lines) + os.Stderr.WriteString("WARNING: Integer retokenization from text is experimental\n") + } + + e := commandVolumes[commandTarget].AppleDOSWriteFile(name, kind, data, int(addr)) + if e != nil { + os.Stderr.WriteString("Failed to create file: " + e.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + + } else if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + addr := int64(0x0801) + name := filepath.Base(args[0]) + ext := strings.Trim(filepath.Ext(name), ".") + reSpecial := regexp.MustCompile("(?i)^(.+)[#](0x[a-fA-F0-9]+)[.]([A-Za-z]+)$") + if reSpecial.MatchString(name) { + m := reSpecial.FindAllStringSubmatch(name, -1) + name = m[0][1] + ext = strings.ToLower(m[0][3]) + addrStr := m[0][2] + addr, _ = strconv.ParseInt(addrStr, 0, 32) + } else { + name = strings.Replace(name, "."+ext, "", -1) + } + + kind := disk.ProDOSFileTypeFromExt(ext) + + if strings.HasSuffix(args[0], ".INT.ASC") { + kind = disk.FileType_PD_INT + } else if strings.HasSuffix(args[0], ".APP.ASC") { + kind = disk.FileType_PD_APP + } + + if kind == disk.FileType_PD_APP && isASCII(data) { + lines := strings.Split(string(data), "\n") + data = disk.ApplesoftTokenize(lines) + } else if kind == disk.FileType_PD_INT && isASCII(data) { + lines := strings.Split(string(data), "\n") + data = disk.IntegerTokenize(lines) + os.Stderr.WriteString("WARNING: Integer retokenization from text is experimental\n") + } + + e := commandVolumes[commandTarget].PRODOSWriteFile(commandPath, name, kind, data, int(addr)) + if e != nil { + os.Stderr.WriteString("Failed to create file: " + e.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + + } else { + os.Stderr.WriteString("Writing files not supported on " + commandVolumes[commandTarget].Format.String()) + return -1 + } + + return 0 + +} + +func shellDelete(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_DOS_SECTORS_13, disk.DF_DOS_SECTORS_16}) { + err = commandVolumes[commandTarget].AppleDOSDeleteFile(args[0]) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + + } else if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + + if strings.Contains(args[0], "/") { + commandPath = filepath.Dir(args[0]) + args[0] = filepath.Base(args[0]) + } + + err = commandVolumes[commandTarget].PRODOSDeleteFile(commandPath, args[0]) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + } else { + os.Stderr.WriteString("Deleting files not supported on " + commandVolumes[commandTarget].Format.String()) + return -1 + } + + return 0 + +} + +func shellIngest(args []string) int { + + processed = 0 + + dskName := args[0] + + info, err := os.Stat(dskName) + if err != nil { + loggy.Get(0).Errorf("Error stating file: %s", err.Error()) + os.Exit(2) + } + if info.IsDir() { + walk(dskName) + } else { + indisk = make(map[disk.DiskFormat]int) + outdisk = make(map[disk.DiskFormat]int) + + panic.Do( + func() { + var e error + _, e = analyze(0, dskName) + // handle any disk specific + if e != nil { + os.Stderr.WriteString("Error processing disk") + } + }, + func(r interface{}) { + loggy.Get(0).Errorf("Error processing volume: %s", dskName) + loggy.Get(0).Errorf(string(debug.Stack())) + }, + ) + } + + return 0 +} + +func shellLock(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_DOS_SECTORS_13, disk.DF_DOS_SECTORS_16}) { + err = commandVolumes[commandTarget].AppleDOSSetLocked(args[0], true) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + + } else if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + + if strings.Contains(args[0], "/") { + commandPath = filepath.Dir(args[0]) + args[0] = filepath.Base(args[0]) + } + + err = commandVolumes[commandTarget].PRODOSSetLocked(commandPath, args[0], true) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + } else { + os.Stderr.WriteString("Locking files not supported on " + commandVolumes[commandTarget].Format.String()) + return -1 + } + + return 0 +} + +func shellUnlock(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + _, err := analyze(0, fullpath) + if err != nil { + return 1 + } + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_DOS_SECTORS_13, disk.DF_DOS_SECTORS_16}) { + err = commandVolumes[commandTarget].AppleDOSSetLocked(args[0], false) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + + } else if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + + if strings.Contains(args[0], "/") { + commandPath = filepath.Dir(args[0]) + args[0] = filepath.Base(args[0]) + } + + err = commandVolumes[commandTarget].PRODOSSetLocked(commandPath, args[0], false) + if err != nil { + os.Stderr.WriteString(err.Error()) + return -1 + } + saveDisk(commandVolumes[commandTarget], fullpath) + } else { + os.Stderr.WriteString("Locking files not supported on " + commandVolumes[commandTarget].Format.String()) + return -1 + } + + return 0 +} + +func shellDisks(args []string) int { + + fmt.Println("Mounted Volumes") + for i, d := range commandVolumes { + if d != nil { + fmt.Printf("%d:%s\n", i, d.Filename) + } + } + + return 0 +} + +func shellPrefix(args []string) int { + + tmp, err := strconv.ParseInt(args[0], 10, 32) + if err != nil { + os.Stderr.WriteString("Invalid slot number: " + args[0] + "\n") + return -1 + } + + slotid := int(tmp) + if slotid < 0 || slotid >= MAXVOL { + os.Stderr.WriteString(fmt.Sprintf("Valid slots are %d to %d.\n", 0, MAXVOL-1)) + return -1 + } + + d := commandVolumes[slotid] + if d == nil { + os.Stderr.WriteString(fmt.Sprintf("Nothing mounted in slot %d (use disks to see mounts)\n", slotid)) + return -1 + } + + commandTarget = slotid + + return 0 + +} + +func shellD2DCopy(args []string) int { + + reCopyArg := regexp.MustCompile("(?i)^(([0-9])[:])?(.+)$") + reCopyTarget := regexp.MustCompile("(?i)^(([0-9])[:])?(.+)?$") + + l := len(args) + sources := args[0 : l-1] + target := args[l-1] + + var allfiles []*DiskFile + + for _, arg := range sources { + if reCopyArg.MatchString(arg) { + m := reCopyArg.FindAllStringSubmatch(arg, -1) + volume := commandTarget + if m[0][2] != "" { + tmp, err := strconv.ParseInt(m[0][2], 10, 32) + if err != nil { + os.Stderr.WriteString("Invalid slot number: " + m[0][2] + "\n") + return -1 + } + volume = int(tmp) + } + patternstr := m[0][3] + files, _ := globDisk(volume, patternstr) + allfiles = append(allfiles, files...) + } + } + + if reCopyTarget.MatchString(target) { + m := reCopyTarget.FindAllStringSubmatch(target, -1) + volume := commandTarget + if m[0][2] != "" { + tmp, err := strconv.ParseInt(m[0][2], 10, 32) + if err != nil { + os.Stderr.WriteString("Invalid slot number: " + m[0][2] + "\n") + return -1 + } + volume = int(tmp) + } + path := m[0][3] + v := commandVolumes[volume] + if v == nil { + os.Stderr.WriteString("Invalid slot number: " + m[0][2] + "\n") + return -1 + } + if !formatIn( + v.Format.ID, + []disk.DiskFormatID{ + disk.DF_DOS_SECTORS_13, + disk.DF_DOS_SECTORS_16, + disk.DF_PRODOS, + disk.DF_PRODOS_800KB, + disk.DF_PRODOS_400KB, + disk.DF_PRODOS_CUSTOM, + }) { + os.Stderr.WriteString("Target volume does not support write.\n") + return -1 + } + if path != "" && len(allfiles) > 1 { + // copy to path + if !formatIn( + v.Format.ID, + []disk.DiskFormatID{ + disk.DF_PRODOS, + disk.DF_PRODOS_800KB, + disk.DF_PRODOS_400KB, + disk.DF_PRODOS_CUSTOM, + }) { + os.Stderr.WriteString("Only prodos supports copy to directory") + return -1 + } + for _, f := range allfiles { + // must be prodos + name := f.Filename + if len(name) > 15 { + name = name[:15] + } + kind := disk.ProDOSFileTypeFromExt(f.Ext) + auxtype := f.LoadAddress + data := f.Data + e := v.PRODOSWriteFile(path, name, kind, data, auxtype) + if e != nil { + os.Stderr.WriteString(fmt.Sprintf("Failed to copy %s: %s\n", name, e.Error())) + return -1 + } + os.Stderr.WriteString(fmt.Sprintf("Copied %s (%d bytes)\n", name, len(data))) + } + } else { + for _, f := range allfiles { + name := f.Filename + if path != "" && len(allfiles) == 1 { + name = path + } + + if formatIn( + v.Format.ID, + []disk.DiskFormatID{ + disk.DF_PRODOS, + disk.DF_PRODOS_800KB, + disk.DF_PRODOS_400KB, + disk.DF_PRODOS_CUSTOM, + }) { + + if len(name) > 15 { + name = name[:15] + } + kind := disk.ProDOSFileTypeFromExt(f.Ext) + auxtype := f.LoadAddress + data := f.Data + e := v.PRODOSWriteFile("", name, kind, data, auxtype) + if e != nil { + os.Stderr.WriteString(fmt.Sprintf("Failed to copy %s: %s\n", name, e.Error())) + return -1 + } + os.Stderr.WriteString(fmt.Sprintf("Copied %s (%d bytes)\n", name, len(data))) + } else { + // DOS + kind := disk.AppleDOSFileTypeFromExt(f.Ext) + auxtype := f.LoadAddress + data := f.Data + e := v.AppleDOSWriteFile(name, kind, data, auxtype) + if e != nil { + os.Stderr.WriteString(fmt.Sprintf("Failed to copy %s: %s\n", name, e.Error())) + return -1 + } + os.Stderr.WriteString(fmt.Sprintf("Copied %s (%d bytes)\n", name, len(data))) + } + } + } + + // here need to publish disk + fullpath, _ := filepath.Abs(v.Filename) + saveDisk(v, fullpath) + + } else { + os.Stderr.WriteString("Invalid target: " + target + "\n") + return -1 + } + + return 0 +} + +func shellRename(args []string) int { + + fullpath, _ := filepath.Abs(commandVolumes[commandTarget].Filename) + + if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_PRODOS, disk.DF_PRODOS_800KB, disk.DF_PRODOS_400KB, disk.DF_PRODOS_CUSTOM}) { + + oldname := filepath.Base(args[0]) + oldpath := filepath.Dir(args[0]) + newname := filepath.Base(args[1]) + + if oldpath == "." { + oldpath = "" + } + + fmt.Println(oldname, newname, oldpath) + + e := commandVolumes[commandTarget].PRODOSRenameFile(oldpath, oldname, newname) + if e != nil { + os.Stderr.WriteString("Unable to rename file: " + e.Error()) + return -1 + } + + } else if formatIn(commandVolumes[commandTarget].Format.ID, []disk.DiskFormatID{disk.DF_DOS_SECTORS_13, disk.DF_DOS_SECTORS_16}) { + oldname := filepath.Base(args[0]) + newname := filepath.Base(args[1]) + + e := commandVolumes[commandTarget].AppleDOSRenameFile(oldname, newname) + if e != nil { + os.Stderr.WriteString("Unable to rename file: " + e.Error()) + return -1 + } + } else { + os.Stderr.WriteString("Rename currently unsupported on " + commandVolumes[commandTarget].Format.String() + "\n") + return -1 + } + + saveDisk(commandVolumes[commandTarget], fullpath) + + return 0 +} + +func globDisk(slotid int, pattern string) ([]*DiskFile, error) { + + var files []*DiskFile + + if commandVolumes[slotid] == nil { + return []*DiskFile(nil), fmt.Errorf("Invalid slotid %d", slotid) + } + + fullpath, _ := filepath.Abs(commandVolumes[slotid].Filename) + + dsk, err := analyze(0, fullpath) + if err != nil { + return []*DiskFile(nil), fmt.Errorf("Problem reading volume") + } + + r := strings.Replace(pattern, ".", "[.]", -1) + r = strings.Replace(r, "?", ".", -1) + r = strings.Replace(r, "*", ".*", -1) + r = "(?i)^" + r + "$" + + rePattern := regexp.MustCompile(r) + + for _, f := range dsk.Files { + if rePattern.MatchString(f.Filename) { + files = append(files, f) + } + } + + return files, nil + +} + +func shellReport(args []string) int { + + switch args[0] { + case "as-dupes": + activeDupeReport(args[1:]) + case "file-dupes": + fileDupeReport(args[1:]) + case "whole-dupes": + wholeDupeReport(args[1:]) + } + + return -1 + +} + +func shellSearch(args []string) int { + + switch args[0] { + case "text": + //activeDupeReport(args[1:]) + searchForTEXT(args[1], args[2:]) + case "filename": + //fileDupeReport(args[1:]) + searchForFilename(args[1], args[2:]) + case "hash": + searchForSHA256(args[1], args[2:]) + } + + return -1 + +} + +func shellQuarantine(args []string) int { + + switch args[0] { + case "as-dupes": + quarantineActiveDisks(args[1:]) + case "whole-dupes": + quarantineWholeDisks(args[1:]) + } + + return -1 + +} + +func moveFile(source, dest string) error { + + source = strings.Replace(source, "\\", "/", -1) + dest = strings.Replace(dest, "\\", "/", -1) + + fmt.Printf("Reading source file: %s\n", source) + data, err := ioutil.ReadFile(source) + if err != nil { + return err + } + + // make sure dest dir actually exists + os.MkdirAll(filepath.Dir(dest), 0755) + + fmt.Printf("Creating dest file: %s\n", dest) + f, err := os.Create(dest) + if err != nil { + return err + } + f.Write(data) + f.Close() + + err = os.Remove(source) + if err != nil { + return err + } + + if _, err := os.Stat(source); err == nil { + fmt.Println(source + " not deleted!!") + return errors.New(source + " not deleted!!") + } + + return nil +} + +func quarantineActiveDisks(filter []string) { + dfc := &DuplicateActiveSectorDiskCollection{} + Aggregate(AggregateDuplicateActiveSectorDisks, dfc, filter) + + reader := bufio.NewReader(os.Stdin) + + for _, list := range dfc.data { + + if len(list) == 1 { + continue + } + + prompt: + + fmt.Println("Which one to keep?") + fmt.Println("(0) Skip this...") + for i, v := range list { + fmt.Printf("(%d) %s\n", i+1, v.Fullpath) + } + fmt.Println() + fmt.Printf("Option (0-%d, q): ", len(list)) + text, _ := reader.ReadString('\n') + + text = strings.ToLower(strings.Trim(text, "\r\n")) + + if text == "q" { + return + } + + if text == "0" { + continue + } + + tmp, _ := strconv.ParseInt(text, 10, 32) + idx := int(tmp) - 1 + + if idx < 0 || idx > len(list) { + goto prompt + } + + for i, v := range list { + if i == idx { + continue + } + path := v.Fullpath + path = strings.Replace(path, ":", "", -1) + path = strings.Replace(path, "\\", "/", -1) + + bpath := binpath() + "/quarantine/" + path + err := moveFile(v.Fullpath, bpath) + if err != nil { + fmt.Println(err) + return + } + + err = moveFile(v.fingerprint, v.fingerprint+".q") + if err != nil { + fmt.Println(err) + return + } + + } + + } +} + +func quarantineWholeDisks(filter []string) { + dfc := &DuplicateWholeDiskCollection{} + Aggregate(AggregateDuplicateWholeDisks, dfc, filter) + + reader := bufio.NewReader(os.Stdin) + + for _, list := range dfc.data { + + if len(list) == 1 { + continue + } + + wprompt: + + fmt.Println("Which one to keep?") + fmt.Println("(0) Skip this...") + for i, v := range list { + fmt.Printf("(%d) %s\n", i+1, v.Fullpath) + } + fmt.Println() + fmt.Printf("Option (0-%d, q): ", len(list)) + text, _ := reader.ReadString('\n') + + text = strings.ToLower(strings.Trim(text, "\r\n")) + + if text == "q" { + return + } + + if text == "0" { + continue + } + + tmp, _ := strconv.ParseInt(text, 10, 32) + idx := int(tmp) - 1 + + if idx < 0 || idx > len(list) { + goto wprompt + } + + for i, v := range list { + if i == idx { + continue + } + path := v.Fullpath + path = strings.Replace(path, ":", "", -1) + path = strings.Replace(path, "\\", "/", -1) + + bpath := binpath() + "/quarantine/" + path + err := moveFile(v.Fullpath, bpath) + if err != nil { + fmt.Println(err) + return + } + + err = moveFile(v.fingerprint, v.fingerprint+".q") + if err != nil { + fmt.Println(err) + return + } + + } + + } +}