fixed #14: add check for nonstandard dos33 sector size and reset if found

* used when restoring segment parser from json

LICENSE

@@ -1,339 +1,373 @@
-		    GNU GENERAL PUBLIC LICENSE
+Mozilla Public License Version 2.0
-		       Version 2, June 1991
+==================================
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+1. Definitions
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+--------------
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
 
-			    Preamble
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
 
-  The licenses for most software are designed to take away your
+1.2. "Contributor Version"
-freedom to share and change it.  By contrast, the GNU General Public
+    means the combination of the Contributions of others (if any) used
-License is intended to guarantee your freedom to share and change free
+    by a Contributor and that particular Contributor's Contribution.
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
 
-  When we speak of free software, we are referring to freedom, not
+1.3. "Contribution"
-price.  Our General Public Licenses are designed to make sure that you
+    means Covered Software of a particular Contributor.
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
 
-  To protect your rights, we need to make restrictions that forbid
+1.4. "Covered Software"
-anyone to deny you these rights or to ask you to surrender the rights.
+    means Source Code Form to which the initial Contributor has attached
-These restrictions translate to certain responsibilities for you if you
+    the notice in Exhibit A, the Executable Form of such Source Code
-distribute copies of the software, or if you modify it.
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
 
-  For example, if you distribute copies of such a program, whether
+1.5. "Incompatible With Secondary Licenses"
-gratis or for a fee, you must give the recipients all the rights that
+    means
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
 
-  We protect your rights with two steps: (1) copyright the software, and
+    (a) that the initial Contributor has attached the notice described
-(2) offer you this license which gives you legal permission to copy,
+        in Exhibit B to the Covered Software; or
-distribute and/or modify the software.
 
-  Also, for each author's protection and ours, we want to make certain
+    (b) that the Covered Software was made available under the terms of
-that everyone understands that there is no warranty for this free
+        version 1.1 or earlier of the License, but not also under the
-software.  If the software is modified by someone else and passed on, we
+        terms of a Secondary License.
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
 
-  Finally, any free program is threatened constantly by software
+1.6. "Executable Form"
-patents.  We wish to avoid the danger that redistributors of a free
+    means any form of the work other than Source Code Form.
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
 
-  The precise terms and conditions for copying, distribution and
+1.7. "Larger Work"
-modification follow.
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
 
-		    GNU GENERAL PUBLIC LICENSE
+1.8. "License"
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+    means this document.
 
-  0. This License applies to any program or other work which contains
+1.9. "Licensable"
-a notice placed by the copyright holder saying it may be distributed
+    means having the right to grant, to the maximum extent possible,
-under the terms of this General Public License.  The "Program", below,
+    whether at the time of the initial grant or subsequently, any and
-refers to any such program or work, and a "work based on the Program"
+    all of the rights conveyed by this License.
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
 
-Activities other than copying, distribution and modification are not
+1.10. "Modifications"
-covered by this License; they are outside its scope.  The act of
+    means any of the following:
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
 
-  1. You may copy and distribute verbatim copies of the Program's
+    (a) any file in Source Code Form that results from an addition to,
-source code as you receive it, in any medium, provided that you
+        deletion from, or modification of the contents of Covered
-conspicuously and appropriately publish on each copy an appropriate
+        Software; or
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
 
-You may charge a fee for the physical act of transferring a copy, and
+    (b) any new file in Source Code Form that contains any Covered
-you may at your option offer warranty protection in exchange for a fee.
+        Software.
 
-  2. You may modify your copy or copies of the Program or any portion
+1.11. "Patent Claims" of a Contributor
-of it, thus forming a work based on the Program, and copy and
+    means any patent claim(s), including without limitation, method,
-distribute such modifications or work under the terms of Section 1
+    process, and apparatus claims, in any patent Licensable by such
-above, provided that you also meet all of these conditions:
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
 
-    a) You must cause the modified files to carry prominent notices
+1.12. "Secondary License"
-    stating that you changed the files and the date of any change.
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
 
-    b) You must cause any work that you distribute or publish, that in
+1.13. "Source Code Form"
-    whole or in part contains or is derived from the Program or any
+    means the form of the work preferred for making modifications.
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
 
-    c) If the modified program normally reads commands interactively
+1.14. "You" (or "Your")
-    when run, you must cause it, when started running for such
+    means an individual or a legal entity exercising rights under this
-    interactive use in the most ordinary way, to print or display an
+    License. For legal entities, "You" includes any entity that
-    announcement including an appropriate copyright notice and a
+    controls, is controlled by, or is under common control with You. For
-    notice that there is no warranty (or else, saying that you provide
+    purposes of this definition, "control" means (a) the power, direct
-    a warranty) and that users may redistribute the program under
+    or indirect, to cause the direction or management of such entity,
-    these conditions, and telling the user how to view a copy of this
+    whether by contract or otherwise, or (b) ownership of more than
-    License.  (Exception: if the Program itself is interactive but
+    fifty percent (50%) of the outstanding shares or beneficial
-    does not normally print such an announcement, your work based on
+    ownership of such entity.
-    the Program is not required to print an announcement.)
 
-These requirements apply to the modified work as a whole.  If
+2. License Grants and Conditions
-identifiable sections of that work are not derived from the Program,
+--------------------------------
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
 
-Thus, it is not the intent of this section to claim rights or contest
+2.1. Grants
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
 
-In addition, mere aggregation of another work not based on the Program
+Each Contributor hereby grants You a world-wide, royalty-free,
-with the Program (or with a work based on the Program) on a volume of
+non-exclusive license:
-a storage or distribution medium does not bring the other work under
-the scope of this License.
 
-  3. You may copy and distribute the Program (or a work based on it,
+(a) under intellectual property rights (other than patent or trademark)
-under Section 2) in object code or executable form under the terms of
+    Licensable by such Contributor to use, reproduce, make available,
-Sections 1 and 2 above provided that you also do one of the following:
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
 
-    a) Accompany it with the complete corresponding machine-readable
+(b) under Patent Claims of such Contributor to make, use, sell, offer
-    source code, which must be distributed under the terms of Sections
+    for sale, have made, import, and otherwise transfer either its
-    1 and 2 above on a medium customarily used for software interchange; or,
+    Contributions or its Contributor Version.
 
-    b) Accompany it with a written offer, valid for at least three
+2.2. Effective Date
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
 
-    c) Accompany it with the information you received as to the offer
+The licenses granted in Section 2.1 with respect to any Contribution
-    to distribute corresponding source code.  (This alternative is
+become effective for each Contribution on the date the Contributor first
-    allowed only for noncommercial distribution and only if you
+distributes such Contribution.
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
 
-The source code for a work means the preferred form of the work for
+2.3. Limitations on Grant Scope
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
 
-If distribution of executable or object code is made by offering
+The licenses granted in this Section 2 are the only rights granted under
-access to copy from a designated place, then offering equivalent
+this License. No additional rights or licenses will be implied from the
-access to copy the source code from the same place counts as
+distribution or licensing of Covered Software under this License.
-distribution of the source code, even though third parties are not
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
-compelled to copy the source along with the object code.
+Contributor:
 
-  4. You may not copy, modify, sublicense, or distribute the Program
+(a) for any code that a Contributor has removed from Covered Software;
-except as expressly provided under this License.  Any attempt
+    or
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
 
-  5. You are not required to accept this License, since you have not
+(b) for infringements caused by: (i) Your and any other third party's
-signed it.  However, nothing else grants you permission to modify or
+    modifications of Covered Software, or (ii) the combination of its
-distribute the Program or its derivative works.  These actions are
+    Contributions with other software (except as part of its Contributor
-prohibited by law if you do not accept this License.  Therefore, by
+    Version); or
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
 
-  6. Each time you redistribute the Program (or any work based on the
+(c) under Patent Claims infringed by Covered Software in the absence of
-Program), the recipient automatically receives a license from the
+    its Contributions.
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
 
-  7. If, as a consequence of a court judgment or allegation of patent
+This License does not grant any rights in the trademarks, service marks,
-infringement or for any other reason (not limited to patent issues),
+or logos of any Contributor (except as may be necessary to comply with
-conditions are imposed on you (whether by court order, agreement or
+the notice requirements in Section 3.4).
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
 
-If any portion of this section is held invalid or unenforceable under
+2.4. Subsequent Licenses
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
 
-It is not the purpose of this section to induce you to infringe any
+No Contributor makes additional grants as a result of Your choice to
-patents or other property right claims or to contest validity of any
+distribute the Covered Software under a subsequent version of this
-such claims; this section has the sole purpose of protecting the
+License (see Section 10.2) or under the terms of a Secondary License (if
-integrity of the free software distribution system, which is
+permitted under the terms of Section 3.3).
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
 
-This section is intended to make thoroughly clear what is believed to
+2.5. Representation
-be a consequence of the rest of this License.
 
-  8. If the distribution and/or use of the Program is restricted in
+Each Contributor represents that the Contributor believes its
-certain countries either by patents or by copyrighted interfaces, the
+Contributions are its original creation(s) or it has sufficient rights
-original copyright holder who places the Program under this License
+to grant the rights to its Contributions conveyed by this License.
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
 
-  9. The Free Software Foundation may publish revised and/or new versions
+2.6. Fair Use
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
 
-Each version is given a distinguishing version number.  If the Program
+This License is not intended to limit any rights You have under
-specifies a version number of this License which applies to it and "any
+applicable copyright doctrines of fair use, fair dealing, or other
-later version", you have the option of following the terms and conditions
+equivalents.
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
 
-  10. If you wish to incorporate parts of the Program into other free
+2.7. Conditions
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
 
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+3. Responsibilities
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+-------------------
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
 
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+3.1. Distribution of Source Form
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
 
-	    How to Apply These Terms to Your New Programs
+3.2. Distribution of Executable Form
 
-  If you develop a new program, and you want it to be of the greatest
+If You distribute Covered Software in Executable Form then:
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
 
-  To do so, attach the following notices to the program.  It is safest
+(a) such Covered Software must also be made available in Source Code
-to attach them to the start of each source file to most effectively
+    Form, as described in Section 3.1, and You must inform recipients of
-convey the exclusion of warranty; and each file should have at least
+    the Executable Form how they can obtain a copy of such Source Code
-the "copyright" line and a pointer to where the full notice is found.
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
 
-    <one line to give the program's name and a brief idea of what it does.>
+(b) You may distribute such Executable Form under the terms of this
-    Copyright (C) <year>  <name of author>
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
 
-    This program is free software; you can redistribute it and/or modify
+3.3. Distribution of a Larger Work
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
+You may create and distribute a Larger Work under terms of Your choice,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
+provided that You also comply with the requirements of this License for
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+the Covered Software. If the Larger Work is a combination of Covered
-    GNU General Public License for more details.
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
 
-    You should have received a copy of the GNU General Public License along
+3.4. Notices
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-Also add information on how to contact you by electronic and paper mail.
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
 
-If the program is interactive, make it output a short notice like this
+3.5. Application of Additional Terms
-when it starts in an interactive mode:
 
-    Gnomovision version 69, Copyright (C) year name of author
+You may choose to offer, and to charge a fee for, warranty, support,
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+indemnity or liability obligations to one or more recipients of Covered
-    This is free software, and you are welcome to redistribute it
+Software. However, You may do so only on Your own behalf, and not on
-    under certain conditions; type `show c' for details.
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
 
-The hypothetical commands `show w' and `show c' should show the appropriate
+4. Inability to Comply Due to Statute or Regulation
-parts of the General Public License.  Of course, the commands you use may
+---------------------------------------------------
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
 
-You should also get your employer (if you work as a programmer) or your
+If it is impossible for You to comply with any of the terms of this
-school, if any, to sign a "copyright disclaimer" for the program, if
+License with respect to some or all of the Covered Software due to
-necessary.  Here is a sample; alter the names:
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
 
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+5. Termination
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+--------------
 
-  <signature of Ty Coon>, 1 April 1989
+5.1. The rights granted under this License will terminate automatically
-  Ty Coon, President of Vice
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
 
-This General Public License does not permit incorporating your program into
+5.2. If You initiate litigation against any entity by asserting a patent
-proprietary programs.  If your program is a subroutine library, you may
+infringement claim (excluding declaratory judgment actions,
-consider it more useful to permit linking proprietary applications with the
+counter-claims, and cross-claims) alleging that a Contributor Version
-library.  If this is what you want to do, use the GNU Lesser General
+directly or indirectly infringes any patent, then the rights granted to
-Public License instead of this License.
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

MANIFEST.in

@@ -2,8 +2,8 @@ include LICENSE
 include README.rst
 recursive-include scripts *
 include atrcopy/templates/*
-include test_data/dos*atr
+include test_data/*_test?.atr
-include test_data/sd*atr
+include test_data/*_test?.atr.*
 include test_data/*.xex
 include test_data/rebuild.sh
 include test_data/create_binary.py

README.rst

@@ -12,24 +12,29 @@ Prerequisites
 Python
 ------
 
+Starting with ``atrcopy`` 7.0, Python 3.6 is **required**. Python 2 support has
+been dropped. Python 3.7 and beyond will be supported when they are released,
+but 3.6 will probably remain the minimum version. From what I know now of
+future Python versions, I don't plan on requiring any language features beyond
+3.6.
+
 Supported Python versions:
 
-* Python 2.7
+* Python 3.6 (and later)
-* Python 3.5
-* Python 3.6
 
-Python 3 compatibility was added in ``atrcopy`` 5.0, but support for Python 3.4
+If you need Python 2 support, ``atrcopy`` 6.5 and earlier supports Python 2.7,
-and older is not planned.
+which you can install with ``pip install "atrcopy<7.0"``
 
 Dependencies
 ------------
 
-Starting with ``atrcopy`` 2.0, `numpy <http://www.numpy.org/>`_ is required. It
+* numpy
-will be automatically installed when installing ``atrcopy`` with ``pip`` as
+
+It will be automatically installed when installing ``atrcopy`` with ``pip`` as
 described below.
 
-It also uses the pure-Python ``future`` compatibility library to help support
+For development, pytest is used to run the test suite, but this is not required
-both Python 2 and Python 3.
+for normal installation of ``atrcopy``.
 
 Installation
 ============
@@ -114,6 +119,27 @@ Other Supported Formats
 atari800 ``.car`` format
 
 
+Supported Compression/Container Formats
+---------------------------------------
+
+Starting with ``atrcopy`` 8.0, compressed disk images are supported
+transparently, so any type of disk image compressed with one of the supported
+container formats can be used directly, without first decompressing it before
+running ``atrcopy``.
+
++--------------------+----------+------+-------+------------------------------+
+| Container          | File Ext | Read | Write | Status                       |
++====================+==========+======+=======+==============================+
+| gzip               | .gz      | Yes  | No    | Read only                    |
++--------------------+----------+------+-------+------------------------------+
+| bzip2              | .bz2     | Yes  | No    | Read only                    |
++--------------------+----------+------+-------+------------------------------+
+| lzma               | .xz      | Yes  | No    | Read only                    |
++--------------------+----------+------+-------+------------------------------+
+| Disk Communicator  | .dcm     | No   | No    | Recognized but unimplemented |
++--------------------+----------+------+-------+------------------------------+
+
+
 Usage
 =====
 
@@ -128,6 +154,7 @@ where the available commands include:
 * ``add``: add files to a disk image
 * ``extract``: copy files from the disk image to the local file system
 * ``assemble``: create a binary file from ATasm source, optionally including segments containing raw binary data
+* ``boot``: create a boot disk using various binary data as input
 * ``delete``: delete files from the disk image
 * ``vtoc``: show and manipulate the VTOC for images that support it
 
@@ -153,6 +180,7 @@ abbreviated as shown here::
         add (a)             Add files to the disk image
         create (c)          Create a new disk image
         assemble (s,asm)    Create a new binary file in the disk image
+        boot (b)            Create a bootable disk image
         delete (rm,del)     Delete files from the disk image
         vtoc (v)            Show a formatted display of sectors free in the disk
                             image
@@ -284,6 +312,45 @@ file named ``AUTOBRUN``. ``AUTOBRUN`` doesn't exist in the image, it's for you
 to supply.
 
 
+Creating a Custom Boot Disk
+---------------------------
+
+Blocks of binary data can be combined into a boot disk in either ATR format for
+Atari or DSK format for Apple::
+
+    $ atrcopy boot --help
+    usage: atrcopy DISK_IMAGE boot [-h] [-f] [-s [ASM [ASM ...]]]
+                                   [-d [DATA [DATA ...]]] [-b [OBJ [OBJ ...]]]
+                                   [-r RUN_ADDR]
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -f, --force           allow file overwrites in the disk image
+      -s [ASM [ASM ...]], --asm [ASM [ASM ...]]
+                            source file(s) to assemble using pyatasm
+      -d [DATA [DATA ...]], --data [DATA [DATA ...]]
+                            binary data file(s) to add to assembly, specify as
+                            file@addr. Only a portion of the file may be included;
+                            specify the subset using standard python slice
+                            notation: file[subset]@addr
+      -b [OBJ [OBJ ...]], --obj [OBJ [OBJ ...]], --bload [OBJ [OBJ ...]]
+                            binary file(s) to add to assembly, either executables
+                            or labeled memory dumps (e.g. BSAVE on Apple ][),
+                            parsing each file's binary segments to add to the
+                            resulting disk image at the load address for each
+                            segment
+      -r RUN_ADDR, --run-addr RUN_ADDR, --brun RUN_ADDR
+                            run address of binary file if not the first byte of
+                            the first segment
+
+One of ``-s``, ``-d``, or ``-b`` must be speficied to provide the source for
+the boot disk. The ``-b`` argument can take an Atari binary in XEX format, and
+will properly handle multiple segments within that file. If no starting address
+is supplied (or, if using an XEX, to override the start address normally
+contained within the XEX), use the ``-r`` option. Otherwise, the run address
+will point to the first byte of the first binary segment.
+
+
 Creating Programs on the Disk Image
 -----------------------------------
 

atrcopy/__init__.py

@@ -1,6 +1,3 @@
-from __future__ import print_function
-from __future__ import absolute_import
-from builtins import str
 import os
 import sys
 import zlib
@@ -9,22 +6,24 @@ import json
 import logging
 log = logging.getLogger(__name__)
 
-from ._metadata import __version__
+from ._version import __version__
 
 try:
     import numpy as np
 except ImportError:
     raise RuntimeError("atrcopy %s requires numpy" % __version__)
 
-from .errors import *
+from . import errors
 from .ataridos import AtrHeader, AtariDosDiskImage, BootDiskImage, AtariDosFile, XexContainerSegment, get_xex, add_atr_header
 from .dos33 import Dos33DiskImage
 from .kboot import KBootImage, add_xexboot_header
 from .segments import SegmentData, SegmentSaver, DefaultSegment, EmptySegment, ObjSegment, RawSectorsSegment, SegmentedFileSegment, user_bit_mask, match_bit_mask, comment_bit_mask, data_style, selected_bit_mask, diff_bit_mask, not_user_bit_mask, interleave_segments, SegmentList, get_style_mask, get_style_bits
 from .spartados import SpartaDosDiskImage
-from .cartridge import A8CartHeader, AtariCartImage
+from .cartridge import A8CartHeader, AtariCartImage, RomImage
-from .parsers import SegmentParser, DefaultSegmentParser, guess_parser_for_mime, guess_parser_for_system, iter_parsers, iter_known_segment_parsers, mime_parse_order, parsers_for_filename
+from .parsers import SegmentParser, DefaultSegmentParser, guess_parser_by_size, guess_parser_for_mime, guess_parser_for_system, guess_container, iter_parsers, iter_known_segment_parsers, mime_parse_order, parsers_for_filename
+from .magic import guess_detail_for_mime
 from .utils import to_numpy, text_to_int
+from .dummy import LocalFilesystem
 
 
 def process(image, dirent, options):
@@ -55,34 +54,47 @@ def process(image, dirent, options):
         print(dirent)
 
 
-def find_diskimage(filename):
+def find_diskimage_from_data(data, verbose=False):
-    try:
+    data = to_numpy(data)
-        with open(filename, "rb") as fh:
+    parser = None
-            if options.verbose:
+    container = guess_container(data, verbose)
-                print("Loading file %s" % filename)
+    if container is not None:
-            rawdata = SegmentData(fh.read())
+        data = container.unpacked
-            parser = None
+    rawdata = SegmentData(data)
-            for mime in mime_parse_order:
+    mime, parser = guess_parser_by_size(rawdata)
-                if options.verbose:
+    if parser is None:
-                    print("Trying MIME type %s" % mime)
+        for mime in mime_parse_order:
-                parser = guess_parser_for_mime(mime, rawdata, options.verbose)
+            if verbose:
-                if parser is None:
+                print("Trying MIME type %s" % mime)
-                    continue
+            parser = guess_parser_for_mime(mime, rawdata, verbose)
-                if options.verbose:
-                    print("Found parser %s" % parser.menu_name)
-                break
             if parser is None:
-                print("%s: Unknown disk image type" % filename)
+                continue
-    except UnsupportedDiskImage as e:
+            if verbose:
-        print("%s: %s" % (filename, e))
+                print("Found parser %s" % parser.menu_name)
-        return None
+            mime2 = guess_detail_for_mime(mime, rawdata, parser)
-    except IOError as e:
+            if mime != mime2:
-        print("%s: %s" % (filename, e))
+                mime = mime2
-        return None
+                if verbose:
+                    print("Magic signature match: %s" % mime)
+            break
+    if parser is None:
+        raise errors.UnsupportedDiskImage("Unknown disk image type")
+    return parser, mime
+
+
+def find_diskimage(filename, verbose=False):
+    if filename == ".":
+        parser = LocalFilesystem()
+        mime = ""
     else:
-        parser.image.filename = filename
+        with open(filename, "rb") as fh:
-        parser.image.ext = ""
+            if verbose:
-        return parser
+                print("Loading file %s" % filename)
+            data = to_numpy(fh.read())
+        parser, mime = find_diskimage_from_data(data, verbose)
+    parser.image.filename = filename
+    parser.image.ext = ""
+    return parser, mime
 
 
 def extract_files(image, files):
@@ -91,18 +103,25 @@ def extract_files(image, files):
     for name in files:
         try:
             dirent = image.find_dirent(name)
-        except FileNotFound:
+        except errors.FileNotFound:
             print("%s not in %s" % (name, image))
             continue
         output = dirent.filename
         if options.lower:
             output = output.lower()
+        if options.dir:
+            if not os.path.exists(options.dir):
+                os.makedirs(options.dir)
+            output = os.path.join(options.dir, output)
         if not options.dry_run:
             data = image.get_file(dirent)
             if os.path.exists(output) and not options.force:
                 print("skipping %s, file exists. Use -f to overwrite" % output)
                 continue
             print("extracting %s -> %s" % (name, output))
+            if options.text:
+                data = data.replace(b'\x7f', b'\t')
+                data = data.replace(b'\x9b', b'\n')
             with open(output, "wb") as fh:
                 fh.write(data)
         else:
@@ -117,7 +136,7 @@ def save_file(image, name, filetype, data):
         else:
             print("skipping %s, use -f to overwrite" % (name))
             return False
-    except FileNotFound:
+    except errors.FileNotFound:
         pass
     print("copying %s to %s" % (name, image.filename))
     if not options.dry_run:
@@ -134,6 +153,7 @@ def add_files(image, files):
     for name in files:
         with open(name, "rb") as fh:
             data = fh.read()
+        name = os.path.basename(name)
         changed = save_file(image, name, filetype, data)
     if changed:
         image.save()
@@ -144,7 +164,7 @@ def remove_files(image, files):
     for name in files:
         try:
             dirent = image.find_dirent(name)
-        except FileNotFound:
+        except errors.FileNotFound:
             print("%s not in %s" % (name, image))
             continue
         print("removing %s from %s" % (name, image))
@@ -184,14 +204,14 @@ def assemble_segments(source_files, data_files, obj_files, run_addr=""):
         try:
             import pyatasm
         except ImportError:
-            raise AtrError("Please install pyatasm to compile code.")
+            raise errors.AtrError("Please install pyatasm to compile code.")
     changed = False
     segments = SegmentList()
     for name in source_files:
         try:
             asm = pyatasm.Assemble(name)
         except SyntaxError as e:
-            raise AtrError("Assembly error: %s" % e.msg)
+            raise errors.AtrError("Assembly error: %s" % e.msg)
         log.debug("Assembled %s into:" % name)
         for first, last, object_code in asm.segments:
             s = segments.add_segment(object_code, first)
@@ -199,7 +219,7 @@ def assemble_segments(source_files, data_files, obj_files, run_addr=""):
             print("adding %s from %s assembly" % (s, name))
     for name in data_files:
         if "@" not in name:
-            raise AtrError("Data files must include a load address specified with the @ char")
+            raise errors.AtrError("Data files must include a load address specified with the @ char")
         name, addr = name.rsplit("@", 1)
         first = text_to_int(addr)
         log.debug("Adding data file %s at $%04x" % (name, first))
@@ -224,26 +244,38 @@ def assemble_segments(source_files, data_files, obj_files, run_addr=""):
             s = segments.add_segment(data, first)
             log.debug("read data for %s" % s.name)
     for name in obj_files:
-        parser = find_diskimage(name)
+        try:
-        if parser and parser.image:
+            parser, _ = find_diskimage(name, options.verbose)
+        except errors.AtrError as e:
+            print(f"skipping {name}: {e}")
+        else:
             for s in parser.segments:
-                if s.start_addr > 0:
+                if hasattr(s, 'run_address'):
-                    print("adding %s from %s" % (s, name))
+                    if not run_addr:
-                    segments.add_segment(s.data, s.start_addr)
+                        run_addr = s.run_address()
+                    else:
+                        print(f"already have run address {run_addr}; skipping {s.run_address()}")
+                elif s.origin > 0:
+                    print(f"adding {s} from {name}")
+                    segments.add_segment(s.data, s.origin)
     if options.verbose:
         for s in segments:
-            print("%s - %04x)" % (str(s)[:-1], s.start_addr + len(s)))
+            print("%s - %04x)" % (str(s)[:-1], s.origin + len(s)))
     if run_addr:
         try:
             run_addr = text_to_int(run_addr)
-        except ValueError:
+        except (AttributeError, ValueError):
-            run_addr = None
+            # not text, try as integer
+            try:
+                run_addr = int(run_addr)
+            except ValueError:
+                run_addr = None
 
     return segments, run_addr
 
 def assemble(image, source_files, data_files, obj_files, run_addr=""):
     segments, run_addr = assemble_segments(source_files, data_files, obj_files, run_addr)
-    file_data, filetype = image.create_executable_file_image(segments, run_addr)
+    file_data, filetype = image.create_executable_file_image(options.output, segments, run_addr)
     print("total file size: $%x (%d) bytes" % (len(file_data), len(file_data)))
     changed = save_file(image, options.output, filetype, file_data)
     if changed:
@@ -251,7 +283,11 @@ def assemble(image, source_files, data_files, obj_files, run_addr=""):
 
 
 def boot_image(image_name, source_files, data_files, obj_files, run_addr=""):
-    image_cls = parsers_for_filename(image_name)[0]
+    try:
+        image_cls = parsers_for_filename(image_name)[0]
+    except errors.InvalidDiskImage as e:
+        print("%s: %s" % (image_name, e))
+        return None
     segments, run_addr = assemble_segments(source_files, data_files, obj_files, run_addr)
     if segments:
         image = image_cls.create_boot_image(segments, run_addr)
@@ -289,60 +325,81 @@ def get_template_path(rel_path="templates"):
     return template_path
 
 
-def get_template_info():
+def get_template_images(partial=""):
     import glob
-    import textwrap
-    fmt = "  %-14s  %s"
 
     path = get_template_path()
     files = glob.glob(os.path.join(path, "*"))
+    templates = {}
+    for path in files:
+        name = os.path.basename(path)
+        if name.endswith(".inf"):
+            continue
+        if partial not in name:
+            continue
+        try:
+            with open(path + ".inf", "r") as fh:
+                s = fh.read()
+                try:
+                    j = json.loads(s)
+                except ValueError:
+                    continue
+                j['name'] = name
+                j['path'] = path
+                templates[name] = j
+        except IOError:
+            continue
+    return templates
+
+
+def get_template_info():
+    import textwrap
+    fmt = "  %-14s  %s"
+
+    templates = get_template_images()
 
     lines = []
     lines.append("available templates:")
-    for name in sorted(files):
+    for name in sorted(templates.keys()):
-        if name.endswith(".inf"):
+        d = textwrap.wrap(templates[name]["description"], 80 - 1 - 14 - 2 - 2)
-            continue
-        try:
-            with open(name + ".inf", "r") as fh:
-                s = fh.read()
-                j = json.loads(s)
-                description = j["description"]
-        except IOError:
-            description = ""
-        d = textwrap.wrap(description, 80 - 1 - 14 - 2 - 2)
         lines.append(fmt % (os.path.basename(name), d[0]))
         lines.extend([fmt % ("", line) for line in d[1:]])
     return os.linesep.join(lines) + os.linesep
 
 
 def get_template_data(template):
-    path = os.path.join(get_template_path(), template)
+    possibilities = get_template_images(template)
+    if not possibilities:
+        raise errors.InvalidDiskImage("Unknown template disk image %s" % template)
+    if len(possibilities) > 1:
+        raise errors.InvalidDiskImage("Name %s is ambiguous (%d matches: %s)" % (template, len(possibilities), ", ".join(sorted(possibilities.keys()))))
+    name, inf = possibilities.popitem()
+    path = inf['path']
     try:
         with open(path, "rb") as fh:
             data = fh.read()
-    except:
-        raise InvalidDiskImage("Template disk image %s not found" % template)
-    try:
-        with open(path + ".inf", "r") as fh:
-            s = fh.read()
-            inf = json.loads(s)
     except IOError:
-        inf = {"description": ""}
+        raise errors.InvalidDiskImage("Failed reading template file %s" % path)
     return data, inf
 
 
 def create_image(template, name):
     import textwrap
 
-    data, inf = get_template_data(template)
+    try:
-    print("using %s template:\n  %s" % (template, "\n  ".join(textwrap.wrap(inf["description"], 77))))
+        data, inf = get_template_data(template)
+    except errors.InvalidDiskImage as e:
+        info = get_template_info()
+        print("Error: %s\n\n%s" % (e, info))
+        return
+    print("Using template %s:\n  %s" % (inf['name'], "\n  ".join(textwrap.wrap(inf["description"], 77))))
     if not options.dry_run:
         if os.path.exists(name) and not options.force:
             print("skipping %s, use -f to overwrite" % (name))
         else:
             with open(name, "wb") as fh:
                 fh.write(data)
-            parser = find_diskimage(name)
+            parser, _ = find_diskimage(name, options.verbose)
             print("created %s: %s" % (name, str(parser.image)))
             list_files(parser.image, [])
     else:
@@ -431,6 +488,8 @@ def run():
     extract_parser.add_argument("-l", "--lower", action="store_true", default=False, help="convert extracted filenames to lower case")
     #extract_parser.add_argument("-n", "--no-sys", action="store_true", default=False, help="only extract things that look like games (no DOS or .SYS files)")
     extract_parser.add_argument("-e", "--ext", action="store", nargs=1, default=False, help="add the specified extension")
+    extract_parser.add_argument("-d", "--dir", action="store", default=False, help="extract to the specified directory")
+    extract_parser.add_argument("-t", "--text", action="store_true", default=False, help="convert text files to unix-style text files")
     extract_parser.add_argument("-f", "--force", action="store_true", default=False, help="allow file overwrites on local filesystem")
     extract_parser.add_argument("files", metavar="FILENAME", nargs="*", help="if not using the -a/--all option, a file (or list of files) to extract from the disk image.")
 
@@ -523,6 +582,8 @@ def run():
                 args.append('list')
     else:
         disk_image_name = None
+        parser.print_help()
+        sys.exit(1)
 
     # print "parsing: %s" % str(args)
     options = parser.parse_args(args)
@@ -545,10 +606,13 @@ def run():
         obj = options.obj[0] if options.obj else []
         boot_image(disk_image_name, asm, data, obj, options.run_addr)
     else:
-        parser = find_diskimage(disk_image_name)
+        try:
-        if parser and parser.image:
+            parser, mime = find_diskimage(disk_image_name, options.verbose)
+        except (errors.UnsupportedContainer, errors.UnsupportedDiskImage, IOError) as e:
+            print(f"{disk_image_name}: {e}")
+        else:
             if command not in skip_diskimage_summary:
-                print("%s: %s" % (disk_image_name, parser.image))
+                print(f"{disk_image_name}: {parser.image}{' (%s}' % mime if mime and options.verbose else ''}")
             if command == "vtoc":
                 vtoc = parser.image.get_vtoc_object()
                 print(vtoc)
@@ -571,5 +635,3 @@ def run():
                 assemble(parser.image, asm, data, obj, options.run_addr)
             elif command == "segments":
                 print("\n".join([str(a) for a in parser.segments]))
-        else:
-            log.error("Invalid disk image: %s" % disk_image_name)

atrcopy/_metadata.py

@@ -1,4 +1,3 @@
-__version__ = "6.2.0"
 __author__ = "Rob McMullen"
 __author_email__ = "feedback@playermissile.com"
 __url__ = "https://github.com/robmcmullen/atrcopy"

atrcopy/_version.py

@@ -0,0 +1 @@
+__version__ = "10.1"

atrcopy/ataridos.py

@@ -1,14 +1,10 @@
-from __future__ import absolute_import
-from __future__ import division
-from builtins import str
-from builtins import range
-from builtins import object
 import numpy as np
 
-from .errors import *
+from . import errors
-from .diskimages import DiskImageBase, BaseHeader
+from .diskimages import DiskImageBase, BaseHeader, Bootable
 from .segments import SegmentData, EmptySegment, ObjSegment, RawSectorsSegment, DefaultSegment, SegmentedFileSegment, SegmentSaver, get_style_bits
 from .utils import *
+from .executables import get_xex
 
 import logging
 log = logging.getLogger(__name__)
@@ -102,8 +98,7 @@ class AtariDosDirent(Dirent):
         self.parse_raw_dirent(image, bytes)
 
     def __str__(self):
-        # return (b'File #%-2d (%s) %03d %-8s%-3s  %03d' % (self.file_num, self.summary.encode("utf-8"), self.starting_sector, self.basename, self.ext, self.num_sectors)).decode("utf-8")
+        return "File #%-2d (%s) %03d %-8s%-3s  %03d" % (self.file_num, self.summary, self.starting_sector, self.basename.decode("latin1"), self.ext.decode("latin1"), self.num_sectors)
-        return "File #%-2d (%s) %03d %-8s%-3s  %03d" % (self.file_num, self.summary, self.starting_sector, self.basename.decode("utf-8"), self.ext.decode("utf-8"), self.num_sectors)
 
     def __eq__(self, other):
         return self.__class__ == other.__class__ and self.filename == other.filename and self.starting_sector == other.starting_sector and self.num_sectors == other.num_sectors
@@ -111,7 +106,7 @@ class AtariDosDirent(Dirent):
     @property
     def filename(self):
         ext = (b'.' + self.ext) if self.ext else b''
-        return self.basename + ext
+        return (self.basename + ext).decode('latin1')
 
     @property
     def summary(self):
@@ -203,7 +198,7 @@ class AtariDosDirent(Dirent):
 
     def start_read(self, image):
         if not self.is_sane:
-            raise InvalidDirent("Invalid directory entry '%s'" % str(self))
+            raise errors.InvalidDirent("Invalid directory entry '%s'" % str(self))
         self.current_sector = self.starting_sector
         self.current_read = self.num_sectors
         self.sectors_seen = set()
@@ -216,18 +211,18 @@ class AtariDosDirent(Dirent):
     def process_raw_sector(self, image, raw):
         file_num = raw[-3] >> 2
         if file_num != self.file_num:
-            raise FileNumberMismatchError164("Expecting file %d, found %d" % (self.file_num, file_num))
+            raise errors.FileNumberMismatchError164("Expecting file %d, found %d" % (self.file_num, file_num))
         self.sectors_seen.add(self.current_sector)
         next_sector = ((raw[-3] & 0x3) << 8) + raw[-2]
         if next_sector in self.sectors_seen:
-            raise InvalidFile("Bad sector pointer data: attempting to reread sector %d" % next_sector)
+            raise errors.InvalidFile("Bad sector pointer data: attempting to reread sector %d" % next_sector)
         self.current_sector = next_sector
         num_bytes = raw[-1]
         return raw[0:num_bytes], num_bytes
 
     def set_values(self, filename, filetype, index):
         if type(filename) is not bytes:
-            filename = filename.encode("utf-8")
+            filename = filename.encode("latin1")
         if b'.' in filename:
             filename, ext = filename.split(b'.', 1)
         else:
@@ -262,7 +257,17 @@ class XexSegment(ObjSegment):
     savers = [SegmentSaver, XexSegmentSaver]
 
 
-class AtariDosFile(object):
+class RunAddressSegment(ObjSegment):
+    # FIXME: defining run_address as a property doesn't work for some reason.
+    # @property
+    # def run_address(self):
+    #     return self.rawdata[0:2].view(dtype="<u2")[0]
+    def run_address(self):
+        return self.rawdata[0:2].data.view(dtype="<u2")[0]
+
+
+
+class AtariDosFile(Bootable):
     """Parse a binary chunk into segments according to the Atari DOS object
     file format.
     
@@ -305,7 +310,7 @@ class AtariDosFile(object):
                 first = False
                 continue
             elif first:
-                raise InvalidBinaryFile("Object file doesn't start with 0xffff")
+                raise errors.InvalidBinaryFile("Object file doesn't start with 0xffff")
             if _xd: log.debug("header parsing: header=0x%x" % header)
             if len(b[pos:pos + 4]) < 4:
                 self.segments.append(ObjSegment(r[pos:pos + 4], 0, 0, 0, len(b[pos:pos + 4]), "Short Segment Header"))
@@ -313,13 +318,17 @@ class AtariDosFile(object):
             start, end = b[pos:pos + 4].view(dtype='<u2')
             s[style_pos:pos + 4] = get_style_bits(data=True)
             if end < start:
-                raise InvalidBinaryFile("Nonsensical start and end addresses")
+                raise errors.InvalidBinaryFile("Nonsensical start and end addresses")
             count = end - start + 1
             found = len(b[pos + 4:pos + 4 + count])
             if found < count:
                 self.segments.append(ObjSegment(r[pos + 4:pos + 4 + count], pos, pos + 4, start, end, "Incomplete Data"))
                 break
-            self.segments.append(ObjSegment(r[pos + 4:pos + 4 + count], pos, pos + 4, start, end))
+            if start == 0x2e0:
+                segment_cls = RunAddressSegment
+            else:
+                segment_cls = ObjSegment
+            self.segments.append(segment_cls(r[pos + 4:pos + 4 + count], pos, pos + 4, start, end))
             pos += 4 + count
             style_pos = pos
 
@@ -350,7 +359,7 @@ class AtrHeader(BaseHeader):
         if len(bytes) == 16:
             values = bytes.view(dtype=self.format)[0]
             if values[0] != 0x296:
-                raise InvalidAtrHeader
+                raise errors.InvalidAtrHeader("no ATR header magic value")
             self.image_size = (int(values[3]) * 256 * 256 + int(values[1])) * 16
             self.sector_size = int(values[2])
             self.crc = int(values[4])
@@ -358,7 +367,7 @@ class AtrHeader(BaseHeader):
             self.flags = int(values[6])
             self.header_offset = 16
         else:
-            raise InvalidAtrHeader
+            raise errors.InvalidAtrHeader("incorrect AHC header size of %d" % len(bytes))
 
     def __str__(self):
         return "%s Disk Image (size=%d (%dx%dB), crc=%d flags=%d unused=%d)" % (self.file_format, self.image_size, self.max_sectors, self.sector_size, self.crc, self.flags, self.unused)
@@ -406,7 +415,7 @@ class AtrHeader(BaseHeader):
 
     def get_pos(self, sector):
         if not self.sector_is_valid(sector):
-            raise ByteNotInFile166("Sector %d out of range" % sector)
+            raise errors.ByteNotInFile166("Sector %d out of range" % sector)
         if sector <= self.num_initial_sectors:
             pos = self.num_initial_sectors * (sector - 1)
             size = self.initial_sector_size
@@ -420,7 +429,7 @@ class AtrHeader(BaseHeader):
         size = len(image)
         if self.header_offset == 16 or size in [92176, 133136, 184336, 183952]:
             return
-        raise InvalidDiskImage("Uncommon size of ATR file")
+        raise errors.InvalidDiskImage("Uncommon size of ATR file")
 
 
 class XfdHeader(AtrHeader):
@@ -440,10 +449,12 @@ class XfdHeader(AtrHeader):
         size = len(image)
         if size in [92160, 133120, 183936, 184320]:
             return
-        raise InvalidDiskImage("Uncommon size of XFD file")
+        raise errors.InvalidDiskImage("Uncommon size of XFD file")
 
 
 class AtariDosDiskImage(DiskImageBase):
+    default_executable_extension = "XEX"
+
     def __init__(self, *args, **kwargs):
         self.first_vtoc = 360
         self.num_vtoc = 1
@@ -495,7 +506,7 @@ class AtariDosDiskImage(DiskImageBase):
         bytes = self.bytes[0:16]
         try:
             self.header = AtrHeader(bytes)
-        except InvalidAtrHeader:
+        except errors.InvalidAtrHeader:
             self.header = XfdHeader()
 
     def calc_vtoc_code(self):
@@ -527,7 +538,7 @@ class AtariDosDiskImage(DiskImageBase):
         self.assert_valid_sector(self.first_vtoc)
         self.num_vtoc = num
         if num < 0 or num > self.calc_vtoc_code():
-            raise InvalidDiskImage("Invalid number of VTOC sectors: %d" % num)
+            raise errors.InvalidDiskImage("Invalid number of VTOC sectors: %d" % num)
 
         self.total_sectors = values[1]
         self.unused_sectors = values[2]
@@ -663,14 +674,10 @@ class AtariDosDiskImage(DiskImageBase):
             try:
                 binary = AtariDosFile(segment.rawdata)
                 segments_out.extend(binary.segments)
-            except InvalidBinaryFile:
+            except errors.InvalidBinaryFile:
                 log.debug("%s not a binary file; skipping segment generation" % str(segment))
         return segments_out
 
-    def create_executable_file_image(self, segments, run_addr=None):
-        base_segment, user_segments = get_xex(segments, run_addr)
-        return base_segment.data, "XEX"
-
 
 class BootDiskImage(AtariDosDiskImage):
     def __str__(self):
@@ -684,7 +691,7 @@ class BootDiskImage(AtariDosDiskImage):
         i = self.header.header_offset
         flag = b[i:i + 2].view(dtype='<u2')[0]
         if flag == 0xffff:
-            raise InvalidDiskImage("Appears to be an executable")
+            raise errors.InvalidDiskImage("Appears to be an executable")
         nsec = b[i + 1]
         bload = b[i + 2:i + 4].view(dtype='<u2')[0]
 
@@ -695,9 +702,9 @@ class BootDiskImage(AtariDosDiskImage):
         max_size = max_ram - bload
         max_sectors = max_size // self.header.sector_size
         if nsec > max_sectors or nsec < 1:
-            raise InvalidDiskImage("Number of boot sectors out of range (tried %d, max=%d" % (nsec, max_sectors))
+            raise errors.InvalidDiskImage("Number of boot sectors out of range (tried %d, max=%d" % (nsec, max_sectors))
         if bload > (0xc000 - (nsec * self.header.sector_size)):
-            raise InvalidDiskImage("Bad boot load address")
+            raise errors.InvalidDiskImage("Bad boot load address")
 
     def get_boot_sector_info(self):
         pass
@@ -743,64 +750,13 @@ class AtariDiskImage(BootDiskImage):
         return "%s Unidentified Contents" % (self.header)
 
     def check_size(self):
-        print self.header
         if self.header is None:
-            raise ("Not a known Atari disk image format")
+            raise errors.InvalidDiskImage("Not a known Atari disk image format")
 
     def get_boot_segments(self):
         return []
 
 
-def get_xex(segments, run_addr=None):
-    segments_copy = [s for s in segments]  # don't affect the original list!
-    main_segment = None
-    sub_segments = []
-    data_style = get_style_bits(data=True)
-    total = 2
-    runad = False
-    for s in segments:
-        total += 4 + len(s)
-        if s.start_addr == 0x2e0:
-            runad = True
-    if not runad:
-        words = np.empty([1], dtype='<u2')
-        if run_addr:
-            found = False
-            for s in segments:
-                if run_addr >= s.start_addr and run_addr < s.start_addr + len(s):
-                    found = True
-                    break
-            if not found:
-                raise InvalidBinaryFile("Run address points outside data segments")
-        else:
-            run_addr = segments[0].start_addr
-        words[0] = run_addr
-        r = SegmentData(words.view(dtype=np.uint8))
-        s = DefaultSegment(r, 0x2e0)
-        segments_copy[0:0] = [s]
-        total += 6
-    bytes = np.zeros([total], dtype=np.uint8)
-    rawdata = SegmentData(bytes)
-    main_segment = DefaultSegment(rawdata)
-    main_segment.data[0:2] = 0xff  # FFFF header
-    main_segment.style[0:2] = data_style
-    i = 2
-    for s in segments_copy:
-        # create new sub-segment inside new main segment that duplicates the
-        # original segment's data/style
-        new_s = DefaultSegment(rawdata[i:i+4+len(s)], s.start_addr)
-        words = new_s.data[0:4].view(dtype='<u2')
-        words[0] = s.start_addr
-        words[1] = s.start_addr + len(s) - 1
-        new_s.style[0:4] = data_style
-        new_s.data[4:4+len(s)] = s[:]
-        new_s.style[4:4+len(s)] = s.style[:]
-        i += 4 + len(s)
-        new_s.copy_user_data(s, 4)
-        sub_segments.append(new_s)
-    return main_segment, sub_segments
-
-
 def add_atr_header(bytes):
     header = AtrHeader(create=True)
     header.check_size(len(bytes))

atrcopy/cartridge.py

@@ -1,11 +1,8 @@
-from __future__ import absolute_import
-from builtins import str
-from builtins import object
 from collections import defaultdict
 
 import numpy as np
 
-from .errors import *
+from . import errors
 from .segments import SegmentData, EmptySegment, ObjSegment
 from .diskimages import DiskImageBase
 from .utils import to_numpy
@@ -111,10 +108,10 @@ def get_cart(cart_type):
     try:
         return known_cart_types[known_cart_type_map[cart_type]]
     except KeyError:
-        raise InvalidDiskImage("Unsupported cart type %d" % cart_type)
+        raise errors.InvalidDiskImage("Unsupported cart type %d" % cart_type)
 
 
-class A8CartHeader(object):
+class A8CartHeader:
     # Atari Cart format described by https://sourceforge.net/p/atari800/source/ci/master/tree/DOC/cart.txt  NOTE: Big endian!
     format = np.dtype([
         ('magic', '|S4'),
@@ -122,6 +119,7 @@ class A8CartHeader(object):
         ('checksum', '>u4'),
         ('unused','>u4')
         ])
+    nominal_length = format.itemsize
     file_format = "Cart"
 
     def __init__(self, bytes=None, create=False):
@@ -141,7 +139,7 @@ class A8CartHeader(object):
         self.main_origin = 0
         self.possible_types = set()
         if create:
-            self.header_offset = 16
+            self.header_offset = self.nominal_length
             self.check_size(0)
         if bytes is None:
             return
@@ -149,13 +147,13 @@ class A8CartHeader(object):
         if len(bytes) == 16:
             values = bytes.view(dtype=self.format)[0]
             if values[0] != b'CART':
-                raise InvalidCartHeader
+                raise errors.InvalidCartHeader
             self.cart_type = int(values[1])
             self.crc = int(values[2])
-            self.header_offset = 16
+            self.header_offset = self.nominal_length
             self.set_type(self.cart_type)
         else:
-            raise InvalidCartHeader
+            raise errors.InvalidCartHeader
 
     def __str__(self):
         return "%s Cartridge (atari800 type=%d size=%d, %d banks, crc=%d)" % (self.cart_name, self.cart_type, self.cart_size, self.bank_size, self.crc)
@@ -163,8 +161,15 @@ class A8CartHeader(object):
     def __len__(self):
         return self.header_offset
 
+    @property
+    def valid(self):
+        return self.cart_type != -1
+
+    def calc_crc_from_data(self, data):
+        self.crc = 0
+
     def to_array(self):
-        raw = np.zeros([16], dtype=np.uint8)
+        raw = np.zeros([self.nominal_length], dtype=np.uint8)
         values = raw.view(dtype=self.format)[0]
         values[0] = b'CART'
         values[1] = self.cart_type
@@ -192,18 +197,14 @@ class A8CartHeader(object):
     def check_size(self, size):
         self.possible_types = set()
         k, r = divmod(size, 1024)
-        if r == 0 or r == 16:
+        if r == 0 or r == self.nominal_length:
             for i, t in enumerate(known_cart_types):
                 valid_size = t[0]
                 if k == valid_size:
                     self.possible_types.add(i)
 
 
-class AtariCartImage(DiskImageBase):
+class BaseAtariCartImage(DiskImageBase):
-    def __init__(self, rawdata, cart_type, filename=""):
-        self.cart_type = cart_type
-        DiskImageBase.__init__(self, rawdata, filename)
-
     def __str__(self):
         return str(self.header)
 
@@ -211,30 +212,29 @@ class AtariCartImage(DiskImageBase):
         data = self.bytes[0:16]
         try:
             self.header = A8CartHeader(data)
-        except InvalidCartHeader:
+        except errors.InvalidCartHeader:
             self.header = A8CartHeader()
-            self.header.set_type(self.cart_type)
 
     def strict_check(self):
-        if self.header.cart_type != self.cart_type:
+        raise NotImplementedError
-            raise InvalidDiskImage("Cart type doesn't match type defined in header")
 
     def relaxed_check(self):
         if self.header.cart_type != self.cart_type:
             # force the header to be the specified cart type
             self.header = A8CartHeader()
             self.header.set_type(self.cart_type)
+            self.check_size()
 
     def check_size(self):
-        if self.header is None:
+        if not self.header.valid:
             return
         k, rem = divmod((len(self) - len(self.header)), 1024)
-        c = get_cart(self.cart_type)
+        c = get_cart(self.header.cart_type)
         log.debug("checking type=%d, k=%d, rem=%d for %s, %s" % (self.cart_type, k, rem, c[1], c[2]))
         if rem > 0:
-            raise InvalidDiskImage("Cart not multiple of 1K")
+            raise errors.InvalidDiskImage("Cart not multiple of 1K")
         if k != c[2]:
-            raise InvalidDiskImage("Image size %d doesn't match cart type %d size %d" % (k, self.cart_type, c[2]))
+            raise errors.InvalidDiskImage("Image size %d doesn't match cart type %d size %d" % (k, self.cart_type, c[2]))
 
     def parse_segments(self):
         r = self.rawdata
@@ -261,6 +261,47 @@ class AtariCartImage(DiskImageBase):
             segments.append(s)
         return segments
 
+    def create_emulator_boot_segment(self):
+        h = self.header
+        k, rem = divmod(len(self), 1024)
+        if rem == 0:
+            h.calc_crc_from_data(self.bytes)
+            data_with_header = np.empty(len(self) + h.nominal_length, dtype=np.uint8)
+            data_with_header[0:h.nominal_length] = h.to_array()
+            data_with_header[h.nominal_length:] = self.bytes
+            r = SegmentData(data_with_header)
+        else:
+            r = self.rawdata
+        s = ObjSegment(r, 0, 0, self.header.main_origin, name="Cart image")
+        return s
+
+
+class AtariCartImage(BaseAtariCartImage):
+    def __init__(self, rawdata, cart_type, filename=""):
+        c = get_cart(cart_type)
+        self.cart_type = cart_type
+        DiskImageBase.__init__(self, rawdata, filename)
+
+    def strict_check(self):
+        if not self.header.valid:
+            raise errors.InvalidDiskImage("Missing cart header")
+        if self.header.cart_type != self.cart_type:
+            raise errors.InvalidDiskImage("Cart type doesn't match type defined in header")
+
+
+class Atari8bitCartImage(AtariCartImage):
+    def strict_check(self):
+        if "5200" in self.header.cart_name:
+            raise errors.InvalidDiskImage("5200 Carts don't work in the home computers.")
+        AtariCartImage.strict_check(self)
+
+
+class Atari5200CartImage(AtariCartImage):
+    def strict_check(self):
+        if "5200" not in self.header.cart_name:
+            raise errors.InvalidDiskImage("Home computer carts don't work in the 5200.")
+        AtariCartImage.strict_check(self)
+
 
 def add_cart_header(bytes):
     header = A8CartHeader(create=True)
@@ -270,3 +311,45 @@ def add_cart_header(bytes):
     data[0:hlen] = header.to_array()
     data[hlen:] = bytes
     return data
+
+
+class RomImage(DiskImageBase):
+    def __str__(self):
+        return f"{len(self.rawdata) // 1024}k ROM image"
+
+    def read_header(self):
+        self.header = A8CartHeader()
+
+    def strict_check(self):
+        self.check_size()
+
+    def check_size(self):
+        size = len(self)
+        if (size & (size - 1)) != 0:
+            raise errors.InvalidDiskImage("ROM image not a power of 2")
+
+    def parse_segments(self):
+        r = self.rawdata
+        s = ObjSegment(r, 0, 0, self.header.main_origin, name="Main Bank")
+        self.segments = [s]
+
+    def create_emulator_boot_segment(self):
+        s = self.segments[0]
+        if s.origin == 0:
+            return None
+        return s
+
+
+class Atari2600CartImage(RomImage):
+    def __str__(self):
+        return f"{len(self.rawdata) // 1024}k Atari 2600 Cartridge"
+
+
+class Atari2600StarpathImage(RomImage):
+    def __str__(self):
+        return f"{len(self.rawdata) // 1024}k Atari 2600 Starpath Cassette"
+
+
+class VectrexCartImage(RomImage):
+    def __str__(self):
+        return f"{len(self.rawdata) // 1024}k Vectrex Cartridge"

atrcopy/container.py

@@ -0,0 +1,82 @@
+import gzip
+import bz2
+import lzma
+import io
+
+import numpy as np
+
+from . import errors
+from .segments import SegmentData
+from .utils import to_numpy
+
+
+class DiskImageContainer:
+    """Unpacker for disk image compression.
+
+    Disk images may be compressed by any number of techniques. Subclasses of
+    DiskImageContainer implement the `unpack_bytes` method which examines the
+    byte_data argument for the supported compression type, and if valid returns
+    the unpacked bytes to be used in the disk image parsing.
+    """
+    def __init__(self, data):
+        self.unpacked = self.__unpack_raw_data(data)
+
+    def __unpack_raw_data(self, data):
+        raw = data.tobytes()
+        try:
+            unpacked = self.unpack_bytes(raw)
+        except EOFError as e:
+            raise errors.InvalidContainer(e)
+        return to_numpy(unpacked)
+
+    def unpack_bytes(self, byte_data):
+        """Attempt to unpack `byte_data` using this unpacking algorithm.
+
+        `byte_data` is a byte string, and should return a byte string if
+        successfully unpacked. Conversion to a numpy array will take place
+        automatically, outside of this method.
+
+        If the data is not recognized by this subclass, raise an
+        InvalidContainer exception. This signals to the caller that a different
+        container type should be tried.
+
+        If the data is recognized by this subclass but the unpacking algorithm
+        is not implemented, raise an UnsupportedContainer exception. This is
+        different than the InvalidContainer exception because it indicates that
+        the data was indeed recognized by this subclass (despite not being
+        unpacked) and checking further containers is not necessary.
+        """
+        pass
+
+
+class GZipContainer(DiskImageContainer):
+    def unpack_bytes(self, byte_data):
+        try:
+            buf = io.BytesIO(byte_data)
+            with gzip.GzipFile(mode='rb', fileobj=buf) as f:
+                unpacked = f.read()
+        except OSError as e:
+            raise errors.InvalidContainer(e)
+        return unpacked
+
+
+class BZipContainer(DiskImageContainer):
+    def unpack_bytes(self, byte_data):
+        try:
+            buf = io.BytesIO(byte_data)
+            with bz2.BZ2File(buf, mode='rb') as f:
+                unpacked = f.read()
+        except OSError as e:
+            raise errors.InvalidContainer(e)
+        return unpacked
+
+
+class LZMAContainer(DiskImageContainer):
+    def unpack_bytes(self, byte_data):
+        try:
+            buf = io.BytesIO(byte_data)
+            with lzma.LZMAFile(buf, mode='rb') as f:
+                unpacked = f.read()
+        except lzma.LZMAError as e:
+            raise errors.InvalidContainer(e)
+        return unpacked

atrcopy/dcm.py

@@ -0,0 +1,48 @@
+import numpy as np
+
+from . import errors
+from .container import DiskImageContainer
+from .segments import SegmentData
+
+
+class DCMContainer(DiskImageContainer):
+    valid_densities = {
+        0: (720, 128),
+        1: (720, 256),
+        2: (1040, 128),
+    }
+
+    def get_next(self):
+        try:
+            data = self.raw[self.index]
+        except IndexError:
+            raise errors.InvalidContainer("Incomplete DCM file")
+        else:
+            self.index += 1
+        return data
+
+    def unpack_bytes(self, data):
+        self.index = 0
+        self.count = len(data)
+        self.raw = data
+        archive_type = self.get_next()
+        if archive_type == 0xf9 or archive_type == 0xfa:
+            archive_flags = self.get_next()
+            if archive_flags & 0x1f != 1:
+                if archive_type == 0xf9:
+                    raise errors.InvalidContainer("DCM multi-file archive combined in the wrong order")
+                else:
+                    raise errors.InvalidContainer("Expected pass one of DCM archive first")
+            density_flag = (archive_flags >> 5) & 3
+            if density_flag not in self.valid_densities:
+                raise errors.InvalidContainer(f"Unsupported density flag {density_flag} in DCM")
+        else:
+            raise errors.InvalidContainer("Not a DCM file")
+
+        # DCM decoding goes here. Currently, instead of decoding it raises the
+        # UnsupportedContainer exception, which signals to the caller that the
+        # container has been successfully identified but can't be parsed.
+        #
+        # When decoding is supported, return the decoded byte array instead of
+        # this exception.
+        raise errors.UnsupportedContainer("DCM archives are not yet supported")

atrcopy/diskimages.py

@@ -1,12 +1,9 @@
-from __future__ import absolute_import
-from builtins import str
-from builtins import range
-from builtins import object
 import numpy as np
 
-from .errors import *
+from . import errors
 from .segments import SegmentData, EmptySegment, ObjSegment, RawSectorsSegment
 from .utils import *
+from .executables import create_executable_file_data
 
 import logging
 log = logging.getLogger(__name__)
@@ -16,7 +13,7 @@ except NameError:
     _xd = False
 
 
-class BaseHeader(object):
+class BaseHeader:
     file_format = "generic"  # text descriptor of file format
     sector_class = WriteableSector
 
@@ -84,7 +81,7 @@ class BaseHeader(object):
         return track, sector
 
     def check_size(self, size):
-        raise InvalidDiskImage("BaseHeader subclasses need custom checks for size")
+        raise errors.InvalidDiskImage("BaseHeader subclasses need custom checks for size")
 
     def strict_check(self, image):
         pass
@@ -95,7 +92,14 @@ class BaseHeader(object):
         return self.sector_class(self.sector_size, data)
 
 
-class DiskImageBase(object):
+class Bootable:
+    def create_emulator_boot_segment(self):
+        return ObjSegment(self.rawdata, 0, 0, 0)
+
+
+class DiskImageBase(Bootable):
+    default_executable_extension = None
+
     def __init__(self, rawdata, filename="", create=False):
         self.rawdata = rawdata
         self.bytes = self.rawdata.get_data()
@@ -134,11 +138,10 @@ class DiskImageBase(object):
         return Directory
 
     def set_filename(self, filename):
-        if type(filename) is not bytes: filename = filename.encode("utf-8")
+        if '.' in filename:
-        if b'.' in filename:
+            self.filename, self.ext = filename.rsplit('.', 1)
-            self.filename, self.ext = filename.rsplit(b'.', 1)
         else:
-            self.filename, self.ext = filename, b''
+            self.filename, self.ext = filename, ''
 
     def dir(self):
         lines = []
@@ -173,32 +176,31 @@ class DiskImageBase(object):
 
     @classmethod
     def new_header(cls, diskimage, format="ATR"):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def as_new_format(self, format="ATR"):
         """ Create a new disk image in the specified format
         """
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def save(self, filename=""):
         if not filename:
             filename = self.filename
             if self.ext:
-                filename += b'.' + self.ext
+                filename += '.' + self.ext
         if not filename:
             raise RuntimeError("No filename specified for save!")
-        if type(filename) is not bytes: filename = filename.encode("utf-8")
         data = self.bytes[:]
         with open(filename, "wb") as fh:
             data.tofile(fh)
 
     def assert_valid_sector(self, sector):
         if not self.header.sector_is_valid(sector):
-            raise ByteNotInFile166("Sector %d out of range" % sector)
+            raise errors.ByteNotInFile166("Sector %d out of range" % sector)
 
     def check_sane(self):
         if not self.all_sane:
-            raise InvalidDiskImage("Invalid directory entries; may be boot disk")
+            raise errors.InvalidDiskImage("Invalid directory entries; may be boot disk")
 
     def read_header(self):
         return BaseHeader()
@@ -280,11 +282,10 @@ class DiskImageBase(object):
         # check if we've been passed a dirent instead of a filename
         if hasattr(filename, "filename"):
             return filename
-        if type(filename) is not bytes: filename = filename.encode("utf-8")
         for dirent in self.files:
             if filename == dirent.filename:
                 return dirent
-        raise FileNotFound("%s not found on disk" % str(filename))
+        raise errors.FileNotFound("%s not found on disk" % str(filename))
 
     def find_file(self, filename):
         dirent = self.find_dirent(filename)
@@ -292,7 +293,7 @@ class DiskImageBase(object):
 
     def get_file(self, dirent):
         segment = self.get_file_segment(dirent)
-        return segment.tostring()
+        return segment.tobytes()
 
     def get_file_segment(self, dirent):
         pass
@@ -302,17 +303,21 @@ class DiskImageBase(object):
         for dirent in self.files:
             try:
                 segment = self.get_file_segment(dirent)
-            except InvalidFile as e:
+            except errors.InvalidFile as e:
                 segment = EmptySegment(self.rawdata, name=dirent.filename, error=str(e))
             segments.append(segment)
         return segments
 
-    def create_executable_file_image(self, segments, run_addr=None):
+    def create_executable_file_image(self, output_name, segments, run_addr=None):
-        raise NotImplementedError
+        try:
+            data, filetype = create_executable_file_data(output_name, segments, run_addr)
+        except errors.UnsupportedContainer:
+            data, filetype = create_executable_file_data(self.default_executable_extension, segments, run_addr)
+        return data, filetype
 
     @classmethod
     def create_boot_image(self, segments, run_addr=None):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     # file writing methods
 
@@ -348,7 +353,7 @@ class DiskImageBase(object):
             self.write_sector_list(sector_list)
             self.write_sector_list(vtoc)
             self.write_sector_list(directory)
-        except AtrError:
+        except errors.AtrError:
             self.rollback_transaction(state)
             raise
         finally:
@@ -382,7 +387,7 @@ class DiskImageBase(object):
             directory.remove_dirent(self, dirent, vtoc, sector_list)
             self.write_sector_list(vtoc)
             self.write_sector_list(directory)
-        except AtrError:
+        except errors.AtrError:
             self.rollback_transaction(state)
             raise
         finally:
@@ -395,7 +400,7 @@ class DiskImageBase(object):
             for sector_num, pos, size in vtoc.iter_free_sectors():
                 if _xd: log.debug("shredding: sector %s at %d, fill value=%d" % (sector_num, pos, fill_value))
                 self.bytes[pos:pos + size] = fill_value
-        except AtrError:
+        except errors.AtrError:
             self.rollback_transaction(state)
             raise
         finally:

atrcopy/dos33.py

@@ -1,15 +1,10 @@
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from builtins import str
-from builtins import range
-from builtins import object
 import numpy as np
 
-from .errors import *
+from . import errors
-from .diskimages import BaseHeader, DiskImageBase
+from .diskimages import BaseHeader, DiskImageBase, Bootable
 from .utils import Directory, VTOC, WriteableSector, BaseSectorList, Dirent
 from .segments import DefaultSegment, EmptySegment, ObjSegment, RawTrackSectorSegment, SegmentSaver, get_style_bits, SegmentData
+from .executables import get_bsave
 
 import logging
 log = logging.getLogger(__name__)
@@ -170,7 +165,7 @@ class Dos33Dirent(Dirent):
         self.deleted = False
         self.track = 0
         self.sector = 0
-        self.filename = b''
+        self.filename = ""
         self.num_sectors = 0
         self.is_sane = True
         self.current_sector_index = 0
@@ -180,7 +175,7 @@ class Dos33Dirent(Dirent):
         self.parse_raw_dirent(image, bytes)
 
     def __str__(self):
-        return "File #%-2d (%s) %03d %-30s %03d %03d" % (self.file_num, self.summary, self.num_sectors, self.filename.decode("ascii", errors='ignore'), self.track, self.sector)
+        return "File #%-2d (%s) %03d %-30s %03d %03d" % (self.file_num, self.summary, self.num_sectors, self.filename, self.track, self.sector)
 
     def __eq__(self, other):
         return self.__class__ == other.__class__ and self.filename == other.filename and self.track == other.track and self.sector == other.sector and self.num_sectors == other.num_sectors
@@ -232,20 +227,20 @@ class Dos33Dirent(Dirent):
         lines.append("sector map: " + str(self.sector_map))
         return "\n".join(lines)
 
-    def parse_raw_dirent(self, image, bytes):
+    def parse_raw_dirent(self, image, data):
-        if bytes is None:
+        if data is None:
             return
-        values = bytes.view(dtype=self.format)[0]
+        values = data.view(dtype=self.format)[0]
         self.track = values[0]
         if self.track == 0xff:
             self.deleted = True
-            self.track = bytes[0x20]
+            self.track = data[0x20]
         else:
             self.deleted = False
         self.sector = values[1]
         self._file_type = values[2] & 0x7f
         self.locked = values[2] & 0x80
-        self.filename = (bytes[3:0x20] - 0x80).tostring().rstrip()
+        self.filename = (data[3:0x20] - 0x80).tobytes().rstrip().decode("ascii", errors='ignore')
         self.num_sectors = int(values[4])
         self.is_sane = self.sanity_check(image)
 
@@ -256,7 +251,7 @@ class Dos33Dirent(Dirent):
         values[1] = self.sector
         values[2] = self.flag
         n = min(len(self.filename), 30)
-        data[3:3+n] = np.fromstring(self.filename, dtype=np.uint8) | 0x80
+        data[3:3+n] = np.frombuffer(self.filename.encode("ascii"), dtype=np.uint8) | 0x80
         data[3+n:] = ord(' ') | 0x80
         if self.deleted:
             data[0x20] = self.track
@@ -326,7 +321,7 @@ class Dos33Dirent(Dirent):
 
     def start_read(self, image):
         if not self.is_sane:
-            raise InvalidDirent("Invalid directory entry '%s'" % str(self))
+            raise errors.InvalidDirent("Invalid directory entry '%s'" % str(self))
         self.get_track_sector_list(image)
         if _xd: log.debug("start_read: %s, t/s list: %s" % (str(self), str(self.sector_map)))
         self.current_sector_index = 0
@@ -352,8 +347,7 @@ class Dos33Dirent(Dirent):
         return self.filename
 
     def set_values(self, filename, filetype, index):
-        if type(filename) is not bytes: filename = filename.encode("utf-8")
+        self.filename = '%-30s' % filename[0:30]
-        self.filename = b'%-30s' % filename[0:30]
         self._file_type = self.text_to_type.get(filetype, 0x04)
         self.locked = False
         self.deleted = False
@@ -376,7 +370,7 @@ class Dos33Header(BaseHeader):
 
     def check_size(self, size):
         if size != 143360:
-            raise InvalidDiskImage("Incorrect size for DOS 3.3 image")
+            raise errors.InvalidDiskImage("Incorrect size for DOS 3.3 image")
         self.image_size = size
         self.first_vtoc = 17 * 16
         self.num_vtoc = 1
@@ -388,6 +382,8 @@ class Dos33Header(BaseHeader):
 
 
 class Dos33DiskImage(DiskImageBase):
+    default_executable_extension = "BSAVE"
+
     def __init__(self, rawdata, filename=""):
         DiskImageBase.__init__(self, rawdata, filename)
         self.default_filetype = "B"
@@ -415,7 +411,7 @@ class Dos33DiskImage(DiskImageBase):
         data, style = self.get_sectors(0)
         magic = data[0:4]
         if (magic == [1, 56, 176, 3]).all():
-            raise InvalidDiskImage("ProDOS format found; not DOS 3.3 image")
+            raise errors.InvalidDiskImage("ProDOS format found; not DOS 3.3 image")
         swap_order = False
         data, style = self.get_sectors(self.header.first_vtoc)
         if data[3] == 3:
@@ -425,7 +421,7 @@ class Dos33DiskImage(DiskImageBase):
                     log.warning("DOS 3.3 byte swap needed!")
                     swap_order = True
             else:
-                raise InvalidDiskImage("Invalid VTOC location for DOS 3.3")
+                raise errors.InvalidDiskImage("Invalid VTOC location for DOS 3.3")
 
 
     vtoc_type = np.dtype([
@@ -451,6 +447,9 @@ class Dos33DiskImage(DiskImageBase):
         values = data[0:self.vtoc_type.itemsize].view(dtype=self.vtoc_type)[0]
         self.header.first_directory = self.header.sector_from_track(values['cat_track'], values['cat_sector'])
         self.header.sector_size = int(values['sector_size'])
+        if self.header.sector_size != 256:
+            log.warning(f"Nonstandard sector size {self.header.sector_size}; this is likely an error, setting to 256")
+            self.header.sector_size = 256
         self.header.max_sectors = int(values['num_tracks']) * int(values['sectors_per_track'])
         self.header.ts_pairs = int(values['max_pairs'])
         self.header.dos_release = values['dos_release']
@@ -570,7 +569,7 @@ class Dos33DiskImage(DiskImageBase):
         next_sector = self.header.sector_from_track(raw[1], raw[2])
         if _xd: log.debug("checking catalog sector %d, next catalog sector: %d" % (sector_num, next_sector))
         if next_sector == 0:
-            raise NoSpaceInDirectory("No space left in catalog")
+            raise errors.NoSpaceInDirectory("No space left in catalog")
         return sector_num, next_sector
 
     def get_file_segment(self, dirent):
@@ -589,7 +588,7 @@ class Dos33DiskImage(DiskImageBase):
                 addr = dirent.get_binary_start_address(self) - 4 # factor in 4 byte header
             else:
                 addr = 0
-            segment = ObjSegment(raw, 0, 0, start_addr=addr, name=name, verbose_name=verbose_name)
+            segment = ObjSegment(raw, 0, 0, origin=addr, name=name, verbose_name=verbose_name)
             if addr > 0:
                 style = segment.get_style_bits(data=True)
                 segment.style[0:4] = style
@@ -597,46 +596,8 @@ class Dos33DiskImage(DiskImageBase):
             segment = EmptySegment(self.rawdata, name=dirent.filename)
         return segment
 
-    def create_executable_file_image(self, segments, run_addr=None):
-        # Apple 2 executables get executed at the first address loaded. If the
-        # run_addr is not the first byte of the combined data, have to create a
-        # new 3-byte segment with a "JMP run_addr" to go at the beginning
-        origin = 100000000
-        last = -1
 
-        for s in segments:
+class Dos33BinFile(Bootable):
-            origin = min(origin, s.start_addr)
-            last = max(last, s.start_addr + len(s))
-            if _xd: log.debug("contiguous bytes needed: %04x - %04x" % (origin, last))
-        if run_addr and run_addr != origin:
-            # check if run_addr points to some location that has data
-            found = False
-            for s in segments:
-                if run_addr >= s.start_addr and run_addr < s.start_addr + len(s):
-                    found = True
-                    break
-            if not found:
-                raise InvalidBinaryFile("Run address points outside data segments")
-            origin -= 3
-            hi, lo = divmod(run_addr, 256)
-            raw = SegmentData([0x4c, lo, hi])
-            all_segments = [DefaultSegment(raw, start_addr=origin)]
-            all_segments.extend(segments)
-        else:
-            all_segments = segments
-        size = last - origin
-        image = np.zeros([size + 4], dtype=np.uint8)
-        words = image[0:4].view(dtype="<u2")  # always little endian
-        words[0] = origin
-        words[1] = size
-        for s in all_segments:
-            index = s.start_addr - origin + 4
-            print("setting data for $%04x - $%04x at index $%04x" % (s.start_addr, s.start_addr + len(s), index))
-            image[index:index + len(s)] = s.data
-        return image, 'B'
-
-
-class Dos33BinFile(object):
     """Parse a binary chunk into segments according to the DOS 3.3 binary
     dump format
     """
@@ -666,16 +627,21 @@ class Dos33BinFile(object):
         if _xd: log.debug("Initial parsing: size=%d" % self.size)
         if len(b[pos:pos + 4]) == 4:
             start, count = b[pos:pos + 4].view(dtype='<u2')
+            if count != self.size - 4:
+                raise errors.InvalidBinaryFile(f"Extra data after BSAVE segment: file size {self.size}, header specifies {count} bytes")
             s[pos:pos + 4] = get_style_bits(data=True)
             data = b[pos + 4:pos + 4 + count]
             if len(data) == count:
                 name = "BSAVE data" % start
             else:
-                name = "Incomplete data: expected %04x, loaded %04x" % (count, len(data))
+                raise errors.InvalidBinaryFile(f"Incomplete BSAVE data: expected {count}, loaded {len(data)}")
             self.segments.append(ObjSegment(r[pos + 4:pos + 4 + count], pos, pos + 4, start, start + len(data), name))
 
         else:
-            self.segments.append(ObjSegment(r[pos:pos + 4], 0, 0, 0, len(b[pos:pos + 4]), "Short Segment Header"))
+            raise errors.InvalidBinaryFile(f"Invalid BSAVE header")
+
+    def create_emulator_boot_segment(self):
+        return self.segments[0]
 
 
 class ProdosHeader(Dos33Header):
@@ -699,12 +665,12 @@ class ProdosDiskImage(DiskImageBase):
         swap_order = False
         if (magic == [1, 56, 176, 3]).all():
             data, style = self.get_sectors(1)
-            prodos = data[3:9].tostring()
+            prodos = data[3:9].tobytes()
             if prodos == "PRODOS":
                 pass
             else:
                 data, style = self.get_sectors(14)
-                prodos = data[3:9].tostring()
+                prodos = data[3:9].tobytes()
                 if prodos == "PRODOS":
                     swap_order = True
                 else:
@@ -715,7 +681,7 @@ class ProdosDiskImage(DiskImageBase):
                     # https://github.com/RasppleII/a2server but it seems that
                     # more magic bytes might be acceptable?
 
-                    #raise InvalidDiskImage("No ProDOS header info found")
+                    #raise errors.InvalidDiskImage("No ProDOS header info found")
                     pass
-            raise UnsupportedDiskImage("ProDOS format found but not supported")
+            raise errors.UnsupportedDiskImage("ProDOS format found but not supported")
-        raise InvalidDiskImage("Not ProDOS format")
+        raise errors.InvalidDiskImage("Not ProDOS format")

atrcopy/dummy.py

@@ -0,0 +1,44 @@
+import os
+
+import numpy as np
+
+from . import errors
+from .segments import SegmentData, EmptySegment, ObjSegment
+from .diskimages import DiskImageBase
+from .utils import to_numpy
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class LocalFilesystemImage(DiskImageBase):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self, path="."):
+        return f"Local filesystem output to: {self.path}"
+
+    def save(self, filename=""):
+        # This is to save the disk image containing the files on the disk image
+        # to the local disk, which doesn't make sense when the disk image is
+        # the filesystem.
+        pass
+
+    def find_dirent(self, name):
+        path = os.path.join(self.path, name)
+        if os.path.exists(path):
+            return True
+        raise errors.FileNotFound("%s not found on disk" % str(name))
+
+    def write_file(self, name, filetype, data):
+        path = os.path.join(self.path, name)
+        with open(path, "wb") as fh:
+            fh.write(data)
+
+    def delete_file(self, name):
+        pass
+
+
+class LocalFilesystem():
+    def __init__(self, path="."):
+        self.image = LocalFilesystemImage(path)

atrcopy/errors.py

@@ -65,3 +65,11 @@ class NotEnoughSpaceOnDisk(AtrError):
 
 class FileNotFound(AtrError):
     pass
+
+
+class UnsupportedContainer(AtrError):
+    pass
+
+
+class InvalidContainer(AtrError):
+    pass

atrcopy/executables.py

@@ -0,0 +1,112 @@
+import numpy as np
+
+from . import errors
+from .segments import SegmentData, EmptySegment, ObjSegment, RawSectorsSegment, DefaultSegment, SegmentedFileSegment, SegmentSaver, get_style_bits
+from .utils import *
+
+import logging
+log = logging.getLogger(__name__)
+try:  # Expensive debugging
+    _xd = _expensive_debugging
+except NameError:
+    _xd = False
+
+
+def get_xex(segments, run_addr=None):
+    segments_copy = [s for s in segments]  # don't affect the original list!
+    main_segment = None
+    sub_segments = []
+    data_style = get_style_bits(data=True)
+    total = 2
+    runad = False
+    for s in segments:
+        total += 4 + len(s)
+        if s.origin == 0x2e0:
+            runad = True
+    if not runad:
+        words = np.empty([1], dtype='<u2')
+        if run_addr:
+            found = False
+            for s in segments:
+                if run_addr >= s.origin and run_addr < s.origin + len(s):
+                    found = True
+                    break
+            if not found:
+                raise errors.InvalidBinaryFile("Run address points outside data segments")
+        else:
+            run_addr = segments[0].origin
+        words[0] = run_addr
+        r = SegmentData(words.view(dtype=np.uint8))
+        s = DefaultSegment(r, 0x2e0)
+        segments_copy[0:0] = [s]
+        total += 6
+    bytes = np.zeros([total], dtype=np.uint8)
+    rawdata = SegmentData(bytes)
+    main_segment = DefaultSegment(rawdata)
+    main_segment.data[0:2] = 0xff  # FFFF header
+    main_segment.style[0:2] = data_style
+    i = 2
+    for s in segments_copy:
+        # create new sub-segment inside new main segment that duplicates the
+        # original segment's data/style
+        new_s = DefaultSegment(rawdata[i:i+4+len(s)], s.origin)
+        words = new_s.data[0:4].view(dtype='<u2')
+        words[0] = s.origin
+        words[1] = s.origin + len(s) - 1
+        new_s.style[0:4] = data_style
+        new_s.data[4:4+len(s)] = s[:]
+        new_s.style[4:4+len(s)] = s.style[:]
+        i += 4 + len(s)
+        new_s.copy_user_data(s, 4)
+        sub_segments.append(new_s)
+    return main_segment, sub_segments
+
+
+def get_bsave(segments, run_addr=None):
+    # Apple 2 executables get executed at the first address loaded. If the
+    # run_addr is not the first byte of the combined data, have to create a
+    # new 3-byte segment with a "JMP run_addr" to go at the beginning
+    origin = 100000000
+    last = -1
+
+    for s in segments:
+        origin = min(origin, s.origin)
+        last = max(last, s.origin + len(s))
+        if _xd: log.debug("contiguous bytes needed: %04x - %04x" % (origin, last))
+    if run_addr and run_addr != origin:
+        # check if run_addr points to some location that has data
+        found = False
+        for s in segments:
+            if run_addr >= s.origin and run_addr < s.origin + len(s):
+                found = True
+                break
+        if not found:
+            raise errors.InvalidBinaryFile("Run address points outside data segments")
+        origin -= 3
+        hi, lo = divmod(run_addr, 256)
+        raw = SegmentData([0x4c, lo, hi])
+        all_segments = [DefaultSegment(raw, origin=origin)]
+        all_segments.extend(segments)
+    else:
+        all_segments = segments
+    size = last - origin
+    image = np.zeros([size + 4], dtype=np.uint8)
+    words = image[0:4].view(dtype="<u2")  # always little endian
+    words[0] = origin
+    words[1] = size
+    for s in all_segments:
+        index = s.origin - origin + 4
+        log.debug("setting data for $%04x - $%04x at index $%04x" % (s.origin, s.origin + len(s), index))
+        image[index:index + len(s)] = s.data
+    return image
+
+
+def create_executable_file_data(filename, segments, run_addr=None):
+    name = filename.lower()
+    if name.endswith("xex"):
+        base_segment, user_segments = get_xex(segments, run_addr)
+        return base_segment.data, "XEX"
+    elif name.endswith("bin") or name.endswith("bsave"):
+        data = get_bsave(segments, run_addr)
+        return data, "B"
+    raise errors.UnsupportedContainer

atrcopy/kboot.py

@@ -1,9 +1,8 @@
-from __future__ import absolute_import
-from __future__ import division
 import numpy as np
 
-from .errors import *
+from . import errors
-from .ataridos import AtariDosDirent, AtariDosDiskImage, XexSegment
+from .ataridos import AtrHeader, AtariDosDirent, AtariDosDiskImage, XexSegment, get_xex
+from .segments import SegmentData
 
 
 class KBootDirent(AtariDosDirent):
@@ -13,11 +12,11 @@ class KBootDirent(AtariDosDirent):
         self.starting_sector = 4
         self.basename = image.filename
         if not self.basename:
-            self.basename = "KBOOT"
+            self.basename = b"KBOOT"
         if self.basename == self.basename.upper():
-            self.ext = "XEX"
+            self.ext = b"XEX"
         else:
-            self.ext = "xex"
+            self.ext = b"xex"
         start, size = image.header.get_pos(4)
         i = image.header.header_offset + 9
         count = image.bytes[i] + 256 * image.bytes[i+1] + 256 * 256 *image.bytes[i + 2]
@@ -42,7 +41,7 @@ class KBootImage(AtariDosDiskImage):
 
     def check_sane(self):
         if not self.all_sane:
-            raise InvalidDiskImage("Doesn't seem to be KBoot header")
+            raise errors.InvalidDiskImage("Doesn't seem to be KBoot header")
 
     def get_vtoc(self):
         pass
@@ -68,12 +67,26 @@ class KBootImage(AtariDosDiskImage):
     def get_directory_segments(self):
         return []
 
+    @classmethod
+    def create_boot_image(cls, segments, run_addr=None):
+        data_segment, _ = get_xex(segments)
+        payload_bytes = add_xexboot_header(data_segment.data)
+        data_bytes = np.zeros(len(payload_bytes) + 16, np.uint8)
+        data_bytes[16:] = payload_bytes[:]
+        header_bytes = data_bytes[0:16]
+        atr_header = AtrHeader(create=True)
+        atr_header.check_size(len(payload_bytes))
+        atr_header.encode(header_bytes)
+        raw = SegmentData(data_bytes)
+        atr = cls(raw, create=True)
+        return atr
+
 
 xexboot_header = b'\x00\x03\x00\x07\r\x07L\r\x07\x1c[\x00\x00\xa0\x00\x8c\t\x03\x8c\x04\x03\x8cD\x02\x8c\xe2\x02\x8c\xe3\x02\xc8\x84\t\x8c\x01\x03\xce\x06\x03\xa91\x8d\x00\x03\xa9R\x8d\x02\x03\xa9\x80\x8d\x08\x03\xa9\x01\x8d\x05\x03\xa9\xe3\x8d0\x02\x8d\x02\xd4\xa9\x07\x8d1\x02\x8d\x03\xd4\xa9\x00\xaa\x8d\x0b\x03\xa9\x04\x8d\n\x03 \xbc\x07\xca \xa5\x07\x85C \xa5\x07\x85D%C\xc9\xff\xf0\xf0 \xa5\x07\x85E \xa5\x07\x85F \xa5\x07\x91C\xe6C\xd0\x02\xe6D\xa5E\xc5C\xa5F\xe5D\xb0\xeb\xad\xe2\x02\r\xe3\x02\xf0\xc9\x86\x19 \xa2\x07\xa6\x19\xa0\x00\x8c\xe2\x02\x8c\xe3\x02\xf0\xb8l\xe2\x02\xad\t\x07\xd0\x0b\xad\n\x07\xd0\x03l\xe0\x02\xce\n\x07\xce\t\x07\xe0\x80\x90"\xa9@\x8d\x03\x03 Y\xe4\x10\x06\xce\x01\x07\xd0\xf1\x00\xee\n\x03\xd0\x03\xee\x0b\x03\xad\n\x03\x8d\x19\xd0\xa0\x00\xa2\x00\xbd\x00\x01\xe8`pppppF\xf8\x07p\x07ppp\x06p\x06p\x06A\xe3\x07\x00\x00\x00\x00\x00,/!$).\'\x0e\x0e\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00&2/-'
 
 
 def insert_bytes(data, offset, string, color):
-    s = np.fromstring(string.upper(), dtype=np.uint8) - 32  # convert to internal
+    s = np.frombuffer(string.upper(), dtype=np.uint8) - 32  # convert to internal
     s = s | color
     count = len(s)
     tx = offset + (20 - count) // 2
@@ -90,7 +103,7 @@ def add_xexboot_header(bytes, bootcode=None, title=b"DEMO", author=b"an atari us
     paragraphs = padded_size // 16
 
     if bootcode is None:
-        bootcode = np.fromstring(xexboot_header, dtype=np.uint8)
+        bootcode = np.copy(np.frombuffer(xexboot_header, dtype=np.uint8))
     else:
         # don't insert title or author in user supplied bootcode; would have to
         # assume that the user supplied everything desired in their own code!

atrcopy/magic.py

@@ -0,0 +1,73 @@
+import numpy as np
+
+import logging
+log = logging.getLogger(__name__)
+
+
+magic = [
+    {'mime': "application/vnd.atari8bit.atr.getaway_pd",
+     'name': "Getaway Public Domain ATR",
+     'signature': [
+         (slice(8, 10), [0x82, 0x39]),
+         (slice(12, 16), [0x67, 0x21, 0x70, 0x64]),
+         ],
+    },
+
+    {'mime': "application/vnd.atari8bit.xex.getaway",
+     'name': "Getaway XEX",
+     'signature': [
+         (slice(0, 6), [0xff, 0xff, 0x80, 0x2a, 0xff, 0x8a]),
+         ],
+    },
+
+    {'mime': "application/vnd.atari8bit.atr.getaway",
+     'name': "Getaway ATR",
+     'signature': [
+         (slice(0x10, 0x19), [0x00, 0xc1, 0x80, 0x0f, 0xcc, 0x22, 0x18, 0x60, 0x0e]),
+         ],
+    },
+
+    {'mime': "application/vnd.atari8bit.atr.jumpman_level_tester",
+     'name': "Jumpman Level Tester from Omnivore",
+     'signature': [
+         (slice(0, 5), [0x96, 0x02 , 0xd0 , 0x05 , 0x80]),
+         (0x0196 + 0x3f, 0x4c),
+         (0x0196 + 0x48, 0x20),
+         (0x0196 + 0x4b, 0x60),
+         (0x0196 + 0x4c, 0xff),
+         ],
+    },
+
+    {'mime': "application/vnd.atari8bit.atr.jumpman",
+     'name': "Jumpman",
+     'signature': [
+         (slice(0, 5), [0x96, 0x02 , 0x80 , 0x16 , 0x80]),
+         (0x0810 + 0x3f, 0x4c),
+         (0x0810 + 0x48, 0x20),
+         (0x0810 + 0x4b, 0x60),
+         (0x0810 + 0x4c, 0xff),
+         ],
+    },
+]
+
+
+def check_signature(raw, sig):
+    for index, expected in sig:
+        actual = raw.data[index].tolist()
+        if actual == expected:
+            log.debug(" match at %s: %s" % (str(index), str(expected)))
+        if actual != expected:
+            log.debug(" failed at %s: %s != %s" % (str(index), str(expected), str(raw.data[index])))
+            return False
+    return True
+
+
+def guess_detail_for_mime(mime, raw, parser):
+    for entry in magic:
+        if entry['mime'].startswith(mime):
+            log.debug("checking entry for %s" % entry['mime'])
+            if check_signature(raw, entry['signature']):
+                log.debug("found match: %s" % entry['name'])
+                return entry['mime']
+    return mime
+

atrcopy/mame.py

@@ -1,9 +1,8 @@
-from __future__ import absolute_import
 import zipfile
 
 import numpy as np
 
-from .errors import *
+from . import errors
 from .segments import SegmentData, EmptySegment, ObjSegment
 from .diskimages import DiskImageBase
 from .utils import to_numpy
@@ -21,7 +20,7 @@ class MameZipImage(DiskImageBase):
                 self.check_zip_size(zf)
                 self.create_rawdata(zf)
         else:
-            raise InvalidDiskImage("Not a MAME zip file")
+            raise errors.InvalidDiskImage("Not a MAME zip file")
         DiskImageBase.__init__(self, self.rawdata, filename)
 
     def __str__(self):
@@ -40,14 +39,14 @@ class MameZipImage(DiskImageBase):
         for item in zf.infolist():
             _, r = divmod(item.file_size, 16)
             if r > 0:
-                raise InvalidDiskImage("zip entry not 16 byte multiple")
+                raise errors.InvalidDiskImage("zip entry not 16 byte multiple")
 
     def create_rawdata(self, zf):
         roms = []
         segment_info = []
         offset = 0
         for item in zf.infolist():
-            rom = np.fromstring(zf.open(item).read(), dtype=np.uint8)
+            rom = np.frombuffer(zf.open(item).read(), dtype=np.uint8)
             roms.append(rom)
             segment_info.append((offset, item.file_size, item.filename, item.CRC))
             offset += item.file_size

atrcopy/omnivore_loader.py

@@ -0,0 +1,27 @@
+from . import find_diskimage_from_data, errors
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def identify_mime(header, fh):
+    mime_type = None
+    try:
+        fh.seek(0)
+        data = fh.read()
+    except IOError as e:
+        log.debug(f"atrcopy loader: error reading entire file: {e}")
+    else:
+        try:
+            parser, mime_type = find_diskimage_from_data(data, True)
+        except (errors.UnsupportedContainer, errors.UnsupportedDiskImage, IOError) as e:
+            log.debug(f"error in atrcopy parser: {e}")
+        else:
+            log.debug(f"{parser.image}: {mime_type}")
+
+        if mime_type:
+            log.debug(f"atrcopy loader: identified {mime_type}")
+            return dict(mime=mime_type, ext="", atrcopy_parser=parser)
+        else:
+            log.debug(f"atrcopy loader: not recognized")
+    return None

atrcopy/parsers.py

@@ -1,23 +1,26 @@
-from __future__ import absolute_import
+import hashlib
-from builtins import str
+
-from builtins import object
 import numpy as np
 
 from .segments import SegmentData, DefaultSegment
 from .kboot import KBootImage
 from .ataridos import AtariDosDiskImage, BootDiskImage, AtariDosFile, XexContainerSegment, AtariDiskImage
 from .spartados import SpartaDosDiskImage
-from .cartridge import AtariCartImage, get_known_carts
+from .cartridge import AtariCartImage, Atari8bitCartImage, Atari5200CartImage, get_known_carts, RomImage, Atari2600CartImage, Atari2600StarpathImage, VectrexCartImage
 from .mame import MameZipImage
 from .dos33 import Dos33DiskImage, ProdosDiskImage, Dos33BinFile
 from .standard_delivery import StandardDeliveryImage
-from .errors import *
+from . import errors
+from .magic import guess_detail_for_mime
+from . import container
+from .dcm import DCMContainer
+from .signatures import sha1_signatures
 
 import logging
 log = logging.getLogger(__name__)
 
 
-class SegmentParser(object):
+class SegmentParser:
     menu_name = ""
     image_type = None
     container_segment = DefaultSegment
@@ -69,24 +72,32 @@ class SegmentParser(object):
         self.segments.append(self.container_segment(r, 0, name=self.menu_name))
         try:
             log.debug("Trying %s" % self.image_type)
+            log.debug(self.image_type.__mro__)
             self.image = self.get_image(r)
             self.check_image()
             self.image.parse_segments()
-        except UnsupportedDiskImage:
+        except errors.UnsupportedDiskImage:
             raise
-        except AtrError as e:
+        except errors.AtrError as e:
-            raise InvalidSegmentParser(e)
+            raise errors.InvalidSegmentParser(e)
         self.segments.extend(self.image.segments)
 
+    def reconstruct_segments(self, new_rawdata):
+        self.image = self.get_image(new_rawdata)
+        self.segment_data = new_rawdata
+        for s in self.segments:
+            s.reconstruct_raw(new_rawdata)
+
     def get_image(self, r):
+        log.info(f"checking image type {self.image_type}")
         return self.image_type(r)
 
     def check_image(self):
         if self.strict:
             try:
                 self.image.strict_check()
-            except AtrError as e:
+            except errors.AtrError as e:
-                raise InvalidSegmentParser(e)
+                raise errors.InvalidSegmentParser(e)
         else:
             self.image.relaxed_check()
 
@@ -136,9 +147,40 @@ class AtariCartSegmentParser(SegmentParser):
     cart_info = None
 
     def get_image(self, r):
+        log.info(f"checking cart type {self.cart_type}: {self.image_type}")
         return self.image_type(r, self.cart_type)
 
 
+class Atari8bitCartParser(AtariCartSegmentParser):
+    menu_name = "Atari Home Computer Cartridge"
+    image_type = Atari8bitCartImage
+
+
+class Atari5200CartParser(AtariCartSegmentParser):
+    menu_name = "Atari 5200 Cartridge"
+    image_type = Atari5200CartImage
+
+
+class Atari2600CartParser(SegmentParser):
+    menu_name = "Atari 2600 Cartridge"
+    image_type = Atari2600CartImage
+
+
+class Atari2600StarpathParser(SegmentParser):
+    menu_name = "Atari 2600 Starpath Cassette"
+    image_type = Atari2600StarpathImage
+
+
+class VectrexParser(SegmentParser):
+    menu_name = "Vectrex Cartridge"
+    image_type = VectrexCartImage
+
+
+class RomParser(SegmentParser):
+    menu_name = "ROM Image"
+    image_type = RomImage
+
+
 class MameZipParser(SegmentParser):
     menu_name = "MAME ROM Zipfile"
     image_type = MameZipImage
@@ -159,6 +201,59 @@ class ProdosSegmentParser(SegmentParser):
     image_type = ProdosDiskImage
 
 
+known_containers = [
+    container.GZipContainer,
+    container.BZipContainer,
+    container.LZMAContainer,
+    DCMContainer,
+]
+
+
+def guess_container(r, verbose=False):
+    for c in known_containers:
+        if verbose:
+            log.info(f"trying container {c}")
+        try:
+            found = c(r)
+        except errors.InvalidContainer as e:
+            continue
+        else:
+            if verbose:
+                log.info(f"found container {c}")
+            return found
+    log.info(f"image does not appear to be a container.")
+    return None
+
+
+def guess_parser_by_size(r, verbose=False):
+    found = None
+    mime = None
+    size = len(r)
+    if size in sha1_signatures:
+        sha_hash = hashlib.sha1(r.data).digest()
+        log.info(f"{size} in signature database, attempting to match {sha_hash}")
+        try:
+            match = sha1_signatures[size][sha_hash]
+        except KeyError:
+            pass
+        else:
+            mime = match[0]
+            log.info(f"found match: {match}")
+            parsers = mime_parsers[mime]
+            for parser in parsers:
+                try:
+                    found = parser(r, False)
+                    break
+                except errors.InvalidSegmentParser as e:
+                    if verbose:
+                        log.info("parser isn't %s: %s" % (parser.__name__, str(e)))
+                    pass
+        if found is None:
+            log.info(f"no matching signature")
+    else:
+        log.info(f"{size} not found in signature database; skipping sha1 matching")
+    return mime, found
+
 def guess_parser_for_mime(mime, r, verbose=False):
     parsers = mime_parsers[mime]
     found = None
@@ -166,7 +261,7 @@ def guess_parser_for_mime(mime, r, verbose=False):
         try:
             found = parser(r, True)
             break
-        except InvalidSegmentParser as e:
+        except errors.InvalidSegmentParser as e:
             if verbose:
                 log.info("parser isn't %s: %s" % (parser.__name__, str(e)))
             pass
@@ -178,16 +273,24 @@ def guess_parser_for_system(mime_base, r):
         if mime.startswith(mime_base):
             p = guess_parser_for_mime(mime, r)
             if p is not None:
+                mime = guess_detail_for_mime(mime, r, p)
                 return mime, p
     return None, None
 
 
 def iter_parsers(r):
-    for mime in mime_parse_order:
+    container = guess_container(r.data)
-        p = guess_parser_for_mime(mime, r)
+    if container is not None:
-        if p is not None:
+        r = SegmentData(container.unpacked)
-            return mime, p
+    mime, parser = guess_parser_by_size(r)
-    return None, None
+    if parser is None:
+        for mime in mime_parse_order:
+            p = guess_parser_for_mime(mime, r)
+            if p is not None:
+                parser = p
+                mime = guess_detail_for_mime(mime, r, p)
+                break
+    return mime, parser
 
 
 def parsers_for_filename(name):
@@ -197,7 +300,17 @@ def parsers_for_filename(name):
         found = None
         for parser in parsers:
             log.debug("parser: %s = %s" % (mime, parser))
-    matches.append(StandardDeliveryImage)
+    n = name.lower()
+    if n.endswith(".atr"):
+        matches.append(KBootImage)
+    elif n.endswith(".dsk"):
+        matches.append(StandardDeliveryImage)
+    else:
+        try:
+            _, name = name.rsplit(".", 1)
+        except ValueError:
+            pass
+        raise errors.InvalidDiskImage("no disk image formats that match '%s'" % name)
     return matches
 
 
@@ -213,12 +326,26 @@ mime_parsers = {
         XexSegmentParser,
         ],
     "application/vnd.atari8bit.cart": [
+        Atari8bitCartParser,
         ],
-    "application/vnd.atari8bit.5200_cart": [
+    "application/vnd.atari5200.cart": [
+        Atari5200CartParser,
+        ],
+    "application/vnd.atari2600.cart": [
+        Atari2600CartParser,
+        ],
+    "application/vnd.atari2600.starpath": [
+        Atari2600StarpathParser,
+        ],
+    "application/vnd.vectrex": [
+        VectrexParser,
         ],
     "application/vnd.mame_rom": [
         MameZipParser,
         ],
+    "application/vnd.rom": [
+        RomParser,
+        ],
     "application/vnd.apple2.dsk": [
         Dos33SegmentParser,
         ProdosSegmentParser,
@@ -228,22 +355,47 @@ mime_parsers = {
         ],
     }
 
+# Note: Atari 2600 scanning not performed here because it will match everything
 mime_parse_order = [
     "application/vnd.atari8bit.atr",
     "application/vnd.atari8bit.xex",
     "application/vnd.atari8bit.cart",
-    "application/vnd.atari8bit.5200_cart",
+    "application/vnd.atari5200.cart",
     "application/vnd.mame_rom",
     "application/vnd.apple2.dsk",
     "application/vnd.apple2.bin",
+    "application/vnd.rom",
+    ]
+
+# different than the above mime_parse_order, this list is the order in which
+# the mime parsers will appear in a UI. Some, like the vectrex and atari2600
+# parsers, aren't included in the parse order because they will identify
+# many things incorrectly. They are used only when parsing by size and
+# signature.
+mime_display_order = [
+    "application/vnd.atari8bit.atr",
+    "application/vnd.atari8bit.xex",
+    "application/vnd.atari8bit.cart",
+    "application/vnd.atari5200.cart",
+    "application/vnd.atari2600.cart",
+    "application/vnd.atari2600.starpath",
+    "application/vnd.vectrex",
+    "application/vnd.mame_rom",
+    "application/vnd.apple2.dsk",
+    "application/vnd.apple2.bin",
+    "application/vnd.rom",
     ]
 
 pretty_mime = {
     "application/vnd.atari8bit.atr": "Atari 8-bit Disk Image",
     "application/vnd.atari8bit.xex": "Atari 8-bit Executable",
     "application/vnd.atari8bit.cart": "Atari 8-bit Cartridge",
-    "application/vnd.atari8bit.5200_cart":"Atari 5200 Cartridge",
+    "application/vnd.atari5200.cart": "Atari 5200 Cartridge",
+    "application/vnd.atari2600.cart": "Atari 2600 Cartridge",
+    "application/vnd.atari2600.starpath": "Atari 2600 Starpath Cassette",
+    "application/vnd.vectrex": "GCE Vectrex Cartridge",
     "application/vnd.mame_rom": "MAME",
+    "application/vnd.rom": "ROM Image",
     "application/vnd.apple2.dsk": "Apple ][ Disk Image",
     "application/vnd.apple2.bin": "Apple ][ Binary",
 }
@@ -254,20 +406,21 @@ for k in sizes:
     for c in grouped_carts[k]:
         t = c[0]
         name = c[1]
-        kclass = type("AtariCartSegmentParser%d" % t, (AtariCartSegmentParser,), {'cart_type': t, 'cart_info': c, 'menu_name': "%s Cartridge" % name})
         if "5200" in name:
-            key = "application/vnd.atari8bit.5200_cart"
+            key = "application/vnd.atari5200.cart"
+            kclass = type("Atari5200CartSegmentParser%d" % t, (Atari5200CartParser, AtariCartSegmentParser), {'cart_type': t, 'cart_info': c, 'menu_name': "%s Cartridge" % name})
         else:
             key = "application/vnd.atari8bit.cart"
+            kclass = type("Atari8bitCartSegmentParser%d" % t, (Atari8bitCartParser, AtariCartSegmentParser), {'cart_type': t, 'cart_info': c, 'menu_name': "%s Cartridge" % name})
         mime_parsers[key].append(kclass)
 
 
 known_segment_parsers = [DefaultSegmentParser]
-for mime in mime_parse_order:
+for mime in mime_display_order:
     known_segment_parsers.extend(mime_parsers[mime])
 
 
 def iter_known_segment_parsers():
     yield "application/octet-stream", "", [DefaultSegmentParser]
-    for mime in mime_parse_order:
+    for mime in mime_display_order:
         yield mime, pretty_mime[mime], mime_parsers[mime]

atrcopy/segments.py

@@ -1,17 +1,9 @@
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import zip
-from builtins import str
-from builtins import range
-from builtins import object
 import bisect
 import io
 
 import numpy as np
 
-from .errors import *
+from . import errors
 from .utils import to_numpy, to_numpy_list, uuid
 from functools import reduce
 
@@ -65,16 +57,30 @@ def get_style_mask(**kwargs):
     return 0xff ^ bits
 
 
-class SegmentSaver(object):
+class SegmentSaver:
     export_data_name = "Raw Data"
     export_extensions = [".dat"]
 
     @classmethod
     def encode_data(cls, segment, ui_control):
-        return segment.tostring()
+        return segment.tobytes()
 
 
-class OrderWrapper(object):
+class BSAVESaver:
+    export_data_name = "Apple ][ Binary"
+    export_extensions = [".bsave"]
+
+    @classmethod
+    def encode_data(cls, segment, ui_control):
+        data = segment.tobytes()
+        header = np.empty(2, dtype="<u2")
+        header[0] = segment.origin
+        header[1] = len(data)
+        log.debug("binary data: %x bytes at %x" % (header[1], header[0]))
+        return header.tobytes() + segment.tobytes()
+
+
+class OrderWrapper:
     """Wrapper for numpy data so that manipulations can use normal numpy syntax
     and still affect the data according to the byte ordering.
 
@@ -88,6 +94,9 @@ class OrderWrapper(object):
         self.base = data.base  # base array for numpy bounds determination
         self.order = byte_order
 
+    def __str__(self):
+        return f"OrderWrapper at {hex(id(self))} count={len(self)} order={self.order} base: count={len(self.np_data)}"
+
     def __len__(self):
         return np.alen(self.order)
 
@@ -119,11 +128,11 @@ class OrderWrapper(object):
     def unindexed(self):
         return self.np_data[self.order]
 
-    def tostring(self):
+    def tobytes(self):
-        return self.np_data[self.order].tostring()
+        return self.np_data[self.order].tobytes()
 
 
-class UserExtraData(object):
+class UserExtraData:
     def __init__(self):
         self.comments = dict()
         self.user_data = dict()
@@ -131,7 +140,7 @@ class UserExtraData(object):
             self.user_data[i] = dict()
 
 
-class SegmentData(object):
+class SegmentData:
     def __init__(self, data, style=None, extra=None, debug=False, order=None):
         """Storage for raw data
 
@@ -185,7 +194,7 @@ class SegmentData(object):
         newsize = len(base_raw)
         oldsize = len(self.data_base)
         if newsize < oldsize:
-            raise NotImplementedError("Can't truncate yet")
+            raise errors.NotImplementedError("Can't truncate yet")
         if self.is_indexed:
             self.data.np_data = base_raw.data
             self.data.base = base_raw.data.base
@@ -379,15 +388,15 @@ class SegmentData(object):
         return r
 
 
-class DefaultSegment(object):
+class DefaultSegment:
-    savers = [SegmentSaver]
+    savers = [SegmentSaver, BSAVESaver]
     can_resize_default = False
 
-    base_serializable_attributes = ['start_addr', 'error', 'name', 'verbose_name', 'page_size', 'map_width', 'uuid', 'can_resize']
+    base_serializable_attributes = ['origin', 'error', 'name', 'verbose_name', 'page_size', 'map_width', 'uuid', 'can_resize']
     extra_serializable_attributes = []
 
-    def __init__(self, rawdata, start_addr=0, name="All", error=None, verbose_name=None, memory_map=None):
+    def __init__(self, rawdata, origin=0, name="All", error=None, verbose_name=None, memory_map=None):
-        self.start_addr = int(start_addr)  # force python int to decouple from possibly being a numpy datatype
+        self.origin = int(origin)  # force python int to decouple from possibly being a numpy datatype
         self.set_raw(rawdata)
         self.error = error
         self.name = name
@@ -404,6 +413,9 @@ class DefaultSegment(object):
         self.can_resize = self.__class__.can_resize_default
 
     def set_raw(self, rawdata):
+        if type(rawdata) != SegmentData:
+            log.warning(f"data not in SegmentData format: {type(rawdata)}")
+            rawdata = SegmentData(rawdata)
         self.rawdata = rawdata
         self.update_raw_pointers()
 
@@ -449,6 +461,13 @@ class DefaultSegment(object):
         self.rawdata.replace_arrays(container.rawdata)
         self.update_raw_pointers()
 
+    def create_subset(self, new_order, name, verbose_name=""):
+        raw = self.rawdata.get_indexed(new_order)
+        if not verbose_name:
+            verbose_name = name
+        segment = DefaultSegment(raw, name=name, verbose_name=verbose_name)
+        return segment
+
     def __getstate__(self):
         """Custom jsonpickle state save routine
 
@@ -487,6 +506,7 @@ class DefaultSegment(object):
         self.can_resize = state.pop('can_resize', self.__class__.can_resize_default)
         self.restore_missing_serializable_defaults()
         self.__dict__.update(state)
+        self.restore_renamed_serializable_attributes()
 
     def restore_missing_serializable_defaults(self):
         """Hook for the future when extra serializable attributes are added to
@@ -495,6 +515,17 @@ class DefaultSegment(object):
         """
         pass
 
+    def restore_renamed_serializable_attributes(self):
+        """Hook for the future if attributes have been renamed. The old
+        attribute names will have been restored in the __dict__.update in
+        __setstate__, so this routine should move attribute values to their new
+        names.
+        """
+        if hasattr(self, 'start_addr'):
+            self.origin = self.start_addr
+            log.debug(f"moving start_addr to origin: {self.start_addr}")
+            delattr(self, 'start_addr')
+
     def reconstruct_raw(self, rawdata):
         """Reconstruct the pointers to the parent data arrays
 
@@ -522,7 +553,7 @@ class DefaultSegment(object):
             r = r.get_indexed[other.order]
         return r
 
-    def serialize_extra_to_dict(self, mdict):
+    def serialize_session(self, mdict):
         """Save extra metadata to a dict so that it can be serialized
 
         This is not saved by __getstate__ because child segments will point to
@@ -542,7 +573,7 @@ class DefaultSegment(object):
         # pairs
         mdict["comments"] = self.get_sorted_comments()
 
-    def restore_extra_from_dict(self, e):
+    def restore_session(self, e):
         if 'comments' in e:
             for k, v in e['comments']:
                 self.rawdata.extra.comments[k] = v
@@ -567,8 +598,8 @@ class DefaultSegment(object):
                 self.set_style_ranges(e[slot], user=i)
 
     def __str__(self):
-        if self.start_addr > 0:
+        if self.origin > 0:
-            origin = " @ %04x" % (self.start_addr)
+            origin = " @ %04x" % (self.origin)
         else:
             origin = ""
         s = "%s ($%x bytes%s)" % (self.name, len(self), origin)
@@ -616,7 +647,7 @@ class DefaultSegment(object):
         """Get index into base array's raw data, given the address of a byte
         into this segment
         """
-        return self.get_raw_index(addr - self.start_addr)
+        return self.get_raw_index(addr - self.origin)
 
     def get_index_from_base_index(self, base_index):
         """Get index into this array's data given the index into the base array
@@ -630,8 +661,8 @@ class DefaultSegment(object):
             raise IndexError("index %d not in this segment" % base_index)
         return int(index)
 
-    def tostring(self):
+    def tobytes(self):
-        return self.data.tostring()
+        return self.data.tobytes()
 
     def get_style_bits(self, **kwargs):
         return get_style_bits(**kwargs)
@@ -1021,7 +1052,7 @@ class DefaultSegment(object):
         return sorted([[k, v] for k, v in self.rawdata.extra.comments.items()])
 
     def iter_comments_in_segment(self):
-        start = self.start_addr
+        start = self.origin
         start_index = self.get_raw_index(0)
         end_index = self.get_raw_index(len(self.rawdata))
         for k, v in self.rawdata.extra.comments.items():
@@ -1039,14 +1070,14 @@ class DefaultSegment(object):
 
     def label(self, index, lower_case=True):
         if lower_case:
-            return "%04x" % (index + self.start_addr)
+            return "%04x" % (index + self.origin)
         else:
-            return "%04X" % (index + self.start_addr)
+            return "%04X" % (index + self.origin)
 
     @property
     def search_copy(self):
         if self._search_copy is None:
-            self._search_copy = self.data.tostring()
+            self._search_copy = self.data.tobytes()
         return self._search_copy
 
     def compare_segment(self, other_segment):
@@ -1081,14 +1112,14 @@ class EmptySegment(DefaultSegment):
 class ObjSegment(DefaultSegment):
     extra_serializable_attributes = ['metadata_start', 'data_start']
 
-    def __init__(self, rawdata, metadata_start, data_start, start_addr, end_addr=0,  name="", **kwargs):
+    def __init__(self, rawdata, metadata_start, data_start, origin, end_addr=0,  name="", **kwargs):
-        DefaultSegment.__init__(self, rawdata, start_addr, name, **kwargs)
+        DefaultSegment.__init__(self, rawdata, origin, name, **kwargs)
         self.metadata_start = int(metadata_start)
         self.data_start = int(data_start)
 
     def __str__(self):
         count = len(self)
-        s = "%s $%04x-$%04x ($%04x @ $%04x)" % (self.name, self.start_addr, self.start_addr + count, count, self.data_start)
+        s = "%s $%04x-$%04x ($%04x @ $%04x)" % (self.name, self.origin, self.origin + count, count, self.data_start)
         if self.error:
             s += " " + self.error
         return s
@@ -1097,7 +1128,7 @@ class ObjSegment(DefaultSegment):
     def verbose_info(self):
         count = len(self)
         name = self.verbose_name or self.name
-        s = "%s  address range: $%04x-$%04x ($%04x bytes), file index of first byte: $%04x" % (name, self.start_addr, self.start_addr + count, count, self.data_start)
+        s = "%s  address range: $%04x-$%04x ($%04x bytes), file index of first byte: $%04x" % (name, self.origin, self.origin + count, count, self.data_start)
         if self.error:
             s += "  error='%s'" % self.error
         return s
@@ -1204,11 +1235,11 @@ def interleave_segments(segments, num_bytes):
 
 
 class SegmentList(list):
-    def add_segment(self, data, start_addr=0, name=None):
+    def add_segment(self, data, origin=0, name=None):
-        last = start_addr + len(data)
+        last = origin + len(data)
         if name is None:
-            name = "%04x - %04x, size=%04x" % (start_addr, last, len(data))
+            name = "%04x - %04x, size=%04x" % (origin, last, len(data))
         rawdata = SegmentData(data)
-        s = DefaultSegment(rawdata, start_addr, name)
+        s = DefaultSegment(rawdata, origin, name)
         self.append(s)
         return s

atrcopy/spartados.py

@@ -1,9 +1,6 @@
-from __future__ import absolute_import
-from builtins import str
-from builtins import range
 import numpy as np
 
-from .errors import *
+from . import errors
 from .ataridos import AtariDosDirent, AtariDosDiskImage, XexSegment
 from .segments import DefaultSegment, EmptySegment, ObjSegment, RawSectorsSegment, SegmentSaver
 
@@ -41,7 +38,7 @@ class SpartaDosDirent(AtariDosDirent):
         deleted = "d" if self.deleted else "."
         locked = "*" if self.locked else " "
         flags = "%s%s%s%s%s %03d" % (output, subdir, in_use, deleted, locked, self.starting_sector)
-        return "File #%-2d (%s) %-8s%-3s  %8d  %s" % (self.file_num, flags, self.filename, self.ext, self.length, self.str_timestamp)
+        return "File #%-2d (%s) %-8s%-3s  %8d  %s" % (self.file_num, flags, self.basename.decode('latin1'), self.ext.decode('latin1'), self.length, self.str_timestamp)
 
     @property
     def verbose_info(self):
@@ -53,11 +50,11 @@ class SpartaDosDirent(AtariDosDirent):
         if self.locked: flags.append("LOCK")
         return "flags=[%s]" % ", ".join(flags)
 
-    def parse_raw_dirent(self, image, bytes):
+    def parse_raw_dirent(self, image, data):
-        if bytes is None:
+        if data is None:
             return
-        values = bytes.view(dtype=self.format)[0]
+        values = data.view(dtype=self.format)[0]
-        flag = values['status']
+        flag = values[0]
         self.flag = flag
         self.locked = (flag&0x1) > 0
         self.hidden = (flag&0x10) > 0
@@ -66,15 +63,15 @@ class SpartaDosDirent(AtariDosDirent):
         self.deleted = (flag&0b10000) > 0
         self.is_dir = (flag&0b100000) > 0
         self.opened_output = (flag&0b10000000) > 0
-        self.starting_sector = int(values['sector'])
+        self.starting_sector = int(values[1])
-        self.filename = str(values['filename']).rstrip()
+        self.basename = bytes(values[4]).rstrip()
         if self.is_dir:
-            self.ext = ""
+            self.ext = b""
         else:
-            self.ext = str(values['ext']).rstrip()
+            self.ext = bytes(values[5]).rstrip()
-        self.length = 256*256*values['len_h'] + values['len_l']
+        self.length = 256*256*values[3] + values[2]
-        self.date_array = tuple(bytes[17:20])
+        self.date_array = tuple(data[17:20])
-        self.time_array = tuple(bytes[20:23])
+        self.time_array = tuple(data[20:23])
         self.is_sane = self.sanity_check(image)
 
     def sanity_check(self, image):
@@ -93,7 +90,7 @@ class SpartaDosDirent(AtariDosDirent):
     def start_read(self, image):
         if not self.is_sane:
             log.debug("Invalid directory entry '%s', starting_sector=%s" % (str(self), self.starting_sector))
-            raise InvalidDirent("Invalid directory entry '%s'" % str(self))
+            raise errors.InvalidDirent("Invalid directory entry '%s'" % str(self))
         self.sector_map = image.get_sector_map(self.starting_sector)
         self.sector_map_index = 0
         self.length_remaining = self.length
@@ -162,7 +159,7 @@ class SpartaDosDiskImage(AtariDosDiskImage):
         num = self.header.max_sectors
         self.is_sane = self.total_sectors == num and values['first_free'] <= num and self.first_bitmap <= num and self.root_dir <= num and self.fs_version in [0x11, 0x20, 0x21] and self.sector_size != -1
         if not self.is_sane:
-            raise InvalidDiskImage("Invalid SpartaDos parameters in boot header")
+            raise errors.InvalidDiskImage("Invalid SpartaDos parameters in boot header")
 
     def get_vtoc(self):
         pass
@@ -239,10 +236,10 @@ class SpartaDosDiskImage(AtariDosDiskImage):
             else:
                 break
         if len(byte_order) > 0:
-            name = "%s %d@%d %s" % (dirent.get_filename(), dirent.length, dirent.starting_sector, dirent.str_timestamp)
+            name = "%s %d@%d %s" % (dirent.filename, dirent.length, dirent.starting_sector, dirent.str_timestamp)
-            verbose_name = "%s (%d bytes, sector map@%d) %s %s" % (dirent.get_filename(), dirent.length, dirent.starting_sector, dirent.verbose_info, dirent.str_timestamp)
+            verbose_name = "%s (%d bytes, sector map@%d) %s %s" % (dirent.filename, dirent.length, dirent.starting_sector, dirent.verbose_info, dirent.str_timestamp)
             raw = self.rawdata.get_indexed(byte_order)
             segment = DefaultSegment(raw, name=name, verbose_name=verbose_name)
         else:
-            segment = EmptySegment(self.rawdata, name=dirent.get_filename(), error=dirent.str_timestamp)
+            segment = EmptySegment(self.rawdata, name=dirent.filename, error=dirent.str_timestamp)
         return segment

atrcopy/standard_delivery.py

@@ -1,8 +1,6 @@
-from __future__ import absolute_import
-from __future__ import division
 import numpy as np
 
-from .errors import *
+from . import errors
 from .segments import SegmentData
 from .diskimages import BaseHeader, DiskImageBase
 
@@ -22,14 +20,14 @@ class StandardDeliveryHeader(BaseHeader):
         if np.all(data == (0x01, 0xa8, 0xee, 0x06, 0x08)):
             log.debug("Found 48k loader")
         else:
-            raise InvalidDiskImage("No %s boot header" % self.file_format)
+            raise errors.InvalidDiskImage("No %s boot header" % self.file_format)
 
     def __str__(self):
         return "Standard Delivery Boot Disk (size=%d (%dx%dB)" % (self.file_format, self.image_size, self.max_sectors, self.sector_size)
 
     def check_size(self, size):
         if size != 143360:
-            raise InvalidDiskImage("Incorrect size for Standard Delivery image")
+            raise errors.InvalidDiskImage("Incorrect size for Standard Delivery image")
         self.image_size = size
         self.tracks_per_disk = 35
         self.sectors_per_track = 16
@@ -69,14 +67,14 @@ class StandardDeliveryImage(DiskImageBase):
         raw = SegmentData(np.zeros([143360], dtype=np.uint8))
         dsk = cls(raw, create=True)
         if run_addr is None:
-            run_addr = segments[0].start_addr
+            run_addr = segments[0].origin
 
         chunks = []
 
         for s in segments:
             # find size in 256 byte chunks that start on a page boundary
             # since the loader only deals with page boundaries
-            origin = s.start_addr
+            origin = s.origin
             chunk_start, padding = divmod(origin, 256)
             if (chunk_start == 0x20 or chunk_start == 0x40) and padding == 1:
                 show_hgr = False
@@ -167,7 +165,7 @@ from . fstbt import fstbt
 
 def get_fstbt_code(data, address_list, run_addr):
     pointer = len(fstbt)
-    data[0:pointer] = np.fromstring(fstbt, dtype=np.uint8)
+    data[0:pointer] = np.frombuffer(fstbt, dtype=np.uint8)
     hi, lo = divmod(run_addr, 256)
     data[pointer:pointer + 2] = (lo, hi)
     address_list.append(0xc0)  # last sector flag

atrcopy/templates/dos2dd.atr.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Atari 8-bit DOS 2 double density (180K), empty VTOC", "label": "Atari DOS 2 DD (180K) blank image", "ext": "atr"}
+{"type": "new file", "task": "hex_edit", "description": "Atari 8-bit DOS 2 double density (180K), empty VTOC", "label": "Atari DOS 2 DD (180K) blank image", "ext": "atr"}

atrcopy/templates/dos2ed+2.5.atr.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Atari 8-bit DOS 2 enhanced density (130K) DOS 2.5 system disk", "label": "Atari DOS 2.5 ED (130K) system disk", "ext": "atr"}
+{"type": "new file", "task": "hex_edit", "description": "Atari 8-bit DOS 2 enhanced density (130K) DOS 2.5 system disk", "label": "Atari DOS 2.5 ED (130K) system disk", "ext": "atr"}

atrcopy/templates/dos2ed.atr.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Atari 8-bit DOS 2 enhanced density (130K), empty VTOC", "label": "Atari DOS 2 ED (130K) blank image", "ext": "atr"}
+{"type": "new file", "task": "hex_edit", "description": "Atari 8-bit DOS 2 enhanced density (130K), empty VTOC", "label": "Atari DOS 2 ED (130K) blank image", "ext": "atr"}

atrcopy/templates/dos2sd+2.0s.atr.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Atari 8-bit DOS 2 single density (90K) DOS 2.0S system disk", "label": "Atari DOS 2.0S SD (90K) system disk", "ext": "atr"}
+{"type": "new file", "task": "hex_edit", "description": "Atari 8-bit DOS 2 single density (90K) DOS 2.0S system disk", "label": "Atari DOS 2.0S SD (90K) system disk", "ext": "atr"}

atrcopy/templates/dos2sd.atr.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Atari 8-bit DOS 2 single density (90K), empty VTOC", "label": "Atari DOS 2 SD (90K) blank image", "ext": "atr"}
+{"type": "new file", "task": "hex_edit", "description": "Atari 8-bit DOS 2 single density (90K), empty VTOC", "label": "Atari DOS 2 SD (90K) blank image", "ext": "atr"}

atrcopy/templates/dos33.dsk.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Apple ][ DOS 3.3 (140K) standard RWTS, empty VTOC", "label": "Apple DOS 3.3 (140K) blank image", "ext": "dsk"}
+{"type": "new file", "task": "hex_edit", "description": "Apple ][ DOS 3.3 (140K) standard RWTS, empty VTOC", "label": "Apple DOS 3.3 (140K) blank image", "ext": "dsk"}

atrcopy/templates/dos33autobrun.dsk.inf

@@ -1 +1 @@
-{"task": "hex_edit", "description": "Apple ][ DOS 3.3 (140K) disk image for binary program development: HELLO sets fullscreen HGR and calls BRUN on user-supplied AUTOBRUN binary file", "label": "Apple DOS 3.3 (140K) AUTOBRUN image", "ext": "dsk"}
+{"type": "new file", "task": "hex_edit", "description": "Apple ][ DOS 3.3 (140K) disk image for binary program development: HELLO sets fullscreen HGR and calls BRUN on user-supplied AUTOBRUN binary file", "label": "Apple DOS 3.3 (140K) AUTOBRUN image", "ext": "dsk"}

atrcopy/utils.py

@@ -1,14 +1,9 @@
-from __future__ import absolute_import
-from builtins import zip
-from builtins import str
-from builtins import range
-from builtins import object
 import types
 import uuid as stdlib_uuid
 
 import numpy as np
 
-from .errors import *
+from . import errors
 
 import logging
 log = logging.getLogger(__name__)
@@ -34,7 +29,7 @@ def to_numpy(value):
     if type(value) is np.ndarray:
         return value
     elif type(value) is bytes:
-        return np.fromstring(value, dtype=np.uint8)
+        return np.copy(np.frombuffer(value, dtype=np.uint8))
     elif type(value) is list:
     	return np.asarray(value, dtype=np.uint8)
     raise TypeError("Can't convert to numpy data")
@@ -65,7 +60,7 @@ def text_to_int(text, default_base="hex"):
     return value
 
 
-class WriteableSector(object):
+class WriteableSector:
     def __init__(self, sector_size, data=None, num=-1):
         self._sector_num = num
         self._next_sector = 0
@@ -114,7 +109,7 @@ class WriteableSector(object):
         return data[count:]
 
 
-class BaseSectorList(object):
+class BaseSectorList:
     def __init__(self, header):
         self.header = header
         self.sector_size = header.sector_size
@@ -155,7 +150,7 @@ class BaseSectorList(object):
         self.sectors.extend(sectors)
 
 
-class Dirent(object):
+class Dirent:
     """Abstract base class for a directory entry
 
     """
@@ -164,28 +159,28 @@ class Dirent(object):
         self.file_num = file_num
 
     def __eq__(self, other):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def extra_metadata(self, image):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def mark_deleted(self):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def parse_raw_dirent(self, image, bytes):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def encode_dirent(self):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def get_sectors_in_vtoc(self, image):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def start_read(self, image):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def read_sector(self, image):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
 
 class Directory(BaseSectorList):
@@ -210,7 +205,7 @@ class Directory(BaseSectorList):
                     return i
                 used.add(i)
             if self.num_dirents > 0 and (len(used) >= self.num_dirents):
-                raise NoSpaceInDirectory()
+                raise errors.NoSpaceInDirectory()
             i += 1
         else:
             i = 0
@@ -234,7 +229,7 @@ class Directory(BaseSectorList):
             for dirent in list(self.dirents.values()):
                 if filename == dirent.filename:
                     return dirent
-        raise FileNotFound("%s not found on disk" % filename)
+        raise errors.FileNotFound("%s not found on disk" % filename)
 
     def save_dirent(self, image, dirent, vtoc, sector_list):
         vtoc.assign_sector_numbers(dirent, sector_list)
@@ -249,7 +244,7 @@ class Directory(BaseSectorList):
 
     @property
     def dirent_class(self):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def calc_sectors(self, image):
         self.sectors = []
@@ -275,10 +270,10 @@ class Directory(BaseSectorList):
         return self.sector_class(self.sector_size)
 
     def encode_empty(self):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def encode_dirent(self, dirent):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def store_encoded(self, data):
         while True:
@@ -296,7 +291,7 @@ class Directory(BaseSectorList):
         self.set_sector_numbers(image)
 
     def set_sector_numbers(self, image):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
 
 class VTOC(BaseSectorList):
@@ -322,7 +317,7 @@ class VTOC(BaseSectorList):
                 yield i, pos, size
 
     def parse_segments(self, segments):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def assign_sector_numbers(self, dirent, sector_list):
         """ Map out the sectors and link the sectors together
@@ -333,7 +328,7 @@ class VTOC(BaseSectorList):
         num = len(sector_list)
         order = self.reserve_space(num)
         if len(order) != num:
-            raise InvalidFile("VTOC reserved space for %d sectors. Sectors needed: %d" % (len(order), num))
+            raise errors.InvalidFile("VTOC reserved space for %d sectors. Sectors needed: %d" % (len(order), num))
         file_length = 0
         last_sector = None
         for sector, sector_num in zip(sector_list.sectors, order):
@@ -362,10 +357,10 @@ class VTOC(BaseSectorList):
             if _xd: log.debug("Found sector %d free" % num)
             self.sector_map[num] = 0
             return num
-        raise NotEnoughSpaceOnDisk("No space left in VTOC")
+        raise errors.NotEnoughSpaceOnDisk("No space left in VTOC")
 
     def calc_bitmap(self):
-        raise NotImplementedError
+        raise errors.NotImplementedError
 
     def free_sector_list(self, sector_list):
         for sector in sector_list:

gen-sha.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+import os
+import sys
+import hashlib
+from collections import defaultdict
+import pprint
+
+def parse(filename, mime):
+    data = open(filename, 'rb').read()
+    h = hashlib.sha1(data).digest()
+    name = os.path.basename(os.path.splitext(filename)[0])
+    return len(data), h, name
+
+
+
+if __name__ == '__main__':
+    source = "atrcopy/signatures.py"
+    try:
+        with open(source, 'r') as fh:
+            source_text = fh.read()
+    except OSError:
+        source_text = "sha1_signatures = {}"
+    try:
+        exec(source_text)
+    except:
+        raise
+    print(sha1_signatures)
+    mime = sys.argv[1]
+    new_signatures = defaultdict(dict)
+    new_signatures.update(sha1_signatures)
+    for filename in sys.argv[2:]:
+        print(f"parsing {filename}")
+        size, hash_string, name = parse(filename, mime)
+        print(f"{size} {hash_string} {mime} {name}")
+        new_signatures[size][hash_string] = (mime, name)
+    lines = []
+    lines.append("sha1_signatures = {")
+    for k,v in sorted(new_signatures.items()):
+        lines.append(f"{k}: {{")
+        for h,n in sorted(v.items(), key=lambda a:(a[1], a[0])):
+            lines.append(f"  {h}: {n},")
+        lines.append("},")
+    lines.append("}  # end sha1_signatures\n")
+
+    print("\n".join(lines))
+    with open(source, 'w') as fh:
+        fh.write("\n".join(lines))

scripts/atrcopy

@@ -1,6 +1,12 @@
 #!/usr/bin/env python
 
 if __name__ == "__main__":
-    import atrcopy
+    import sys
+    if sys.version_info < (3, 6, 0):
+        print("atrcopy requires Python 3.6 or greater to run; this is Python %s" % ".".join([str(v) for v in sys.version_info[0:2]]))
+        if sys.version_info[0] == 2:
+            print("Python 2 support was dropped with atrcopy 7.0, so you can either use:\n\n    pip install \"atrcopy<7.0\"\n\nto install a version compatible with Python 2, or install Python 3.6 or higher.")
+    else:
+        import atrcopy
     
-    atrcopy.run()
+        atrcopy.run()

setup.py

@@ -5,7 +5,8 @@ try:
 except ImportError:
     from distutils.core import setup
 
-exec(open('atrcopy/_metadata.py').read())
+exec(compile(open('atrcopy/_version.py').read(), 'atrcopy/_version.py', 'exec'))
+exec(compile(open('atrcopy/_metadata.py').read(), 'atrcopy/_metadata.py', 'exec'))
 
 with open("README.rst", "r") as fp:
     long_description = fp.read()
@@ -23,20 +24,19 @@ setup(name="atrcopy",
         packages=["atrcopy"],
         include_package_data=True,
         scripts=scripts,
+        entry_points={"sawx.loaders": 'atrcopy = atrcopy.omnivore_loader'},
         description="Utility to manage file systems on Atari 8-bit (DOS 2) and Apple ][ (DOS 3.3) disk images.",
         long_description=long_description,
-        license="GPL",
+        license="MPL 2.0",
         classifiers=[
-            "Programming Language :: Python :: 2.7",
-            "Programming Language :: Python :: 3.5",
             "Programming Language :: Python :: 3.6",
             "Intended Audience :: Developers",
-            "License :: OSI Approved :: GNU General Public License (GPL)",
+            "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
             "Topic :: Software Development :: Libraries",
             "Topic :: Utilities",
         ],
+        python_requires = '>=3.6',
         install_requires = [
-            "future",
             'numpy',
         ],
         tests_require = [

test/pytest.ini

@@ -1,2 +1,2 @@
-[pytest]
+#[pytest]
-addopts = --cov=atrcopy --cov-report html --cov-report term
+#addopts = --cov=atrcopy --cov-report html --cov-report term

test/test_add_file.py

@@ -4,11 +4,11 @@ import numpy as np
 
 from mock import *
 
-from atrcopy import SegmentData, AtariDosDiskImage, Dos33DiskImage,InvalidBinaryFile
+from atrcopy import SegmentData, AtariDosDiskImage, Dos33DiskImage
-from atrcopy.errors import *
+from atrcopy import errors
 
 
-class BaseFilesystemModifyTest(object):
+class BaseFilesystemModifyTest:
     diskimage_type = None
     sample_data = None
     num_files_in_sample = 0
@@ -17,24 +17,23 @@ class BaseFilesystemModifyTest(object):
         rawdata = SegmentData(self.sample_data.copy())
         self.image = self.diskimage_type(rawdata)
 
-    def check_entries(self, entries, prefix=b"TEST", save=None):
+    def check_entries(self, entries, prefix="TEST", save=None):
-        if type(prefix) is not bytes: prefix = prefix.encode("utf-8")
         orig_num_files = len(self.image.files)
         filenames = []
         count = 1
         for data in entries:
-            filename = b"%s%d.BIN" % (prefix, count)
+            filename = "%s%d.BIN" % (prefix, count)
             self.image.write_file(filename, None, data)
             assert len(self.image.files) == orig_num_files + count
-            data2 = np.fromstring(self.image.find_file(filename), dtype=np.uint8)
+            data2 = np.frombuffer(self.image.find_file(filename), dtype=np.uint8)
             assert np.array_equal(data, data2[0:len(data)])
             count += 1
 
         # loop over them again to make sure data wasn't overwritten
         count = 1
         for data in entries:
-            filename = b"%s%d.BIN" % (prefix, count)
+            filename = "%s%d.BIN" % (prefix, count)
-            data2 = np.fromstring(self.image.find_file(filename), dtype=np.uint8)
+            data2 = np.frombuffer(self.image.find_file(filename), dtype=np.uint8)
             assert np.array_equal(data, data2[0:len(data)])
             count += 1
             filenames.append(filename)
@@ -48,20 +47,20 @@ class BaseFilesystemModifyTest(object):
         assert len(self.image.files) == self.num_files_in_sample
 
         data = np.asarray([0xff, 0xff, 0x00, 0x60, 0x01, 0x60, 1, 2], dtype=np.uint8)
-        self.image.write_file(b"TEST.XEX", None, data)
+        self.image.write_file("TEST.XEX", None, data)
         assert len(self.image.files) == self.num_files_in_sample + 1
 
-        data2 = np.fromstring(self.image.find_file(b"TEST.XEX"), dtype=np.uint8)
+        data2 = np.frombuffer(self.image.find_file("TEST.XEX"), dtype=np.uint8)
         assert np.array_equal(data, data2[0:len(data)])
 
     def test_50k(self):
         assert len(self.image.files) == self.num_files_in_sample
 
         data = np.arange(50*1024, dtype=np.uint8)
-        self.image.write_file(b"RAMP50K.BIN", None, data)
+        self.image.write_file("RAMP50K.BIN", None, data)
         assert len(self.image.files) == self.num_files_in_sample + 1
 
-        data2 = self.image.find_file(b"RAMP50K.BIN")
+        data2 = self.image.find_file("RAMP50K.BIN")
         assert data.tostring() == data2
 
     def test_many_small(self):
@@ -78,17 +77,17 @@ class BaseFilesystemModifyTest(object):
             np.arange(9*1024, dtype=np.uint8),
             np.arange(10*1024, dtype=np.uint8),
             ]
-        self.check_entries(entries, save=b"many_small.atr")
+        self.check_entries(entries, save="many_small.atr")
 
     def test_big_failure(self):
         assert len(self.image.files) == self.num_files_in_sample
 
         data = np.arange(50*1024, dtype=np.uint8)
-        self.image.write_file(b"RAMP50K.BIN", None, data)
+        self.image.write_file("RAMP50K.BIN", None, data)
         assert len(self.image.files) == self.num_files_in_sample + 1
-        with pytest.raises(NotEnoughSpaceOnDisk):
+        with pytest.raises(errors.NotEnoughSpaceOnDisk):
             huge = np.arange(500*1024, dtype=np.uint8)
-            self.image.write_file(b"RAMP500K.BIN", None, huge)
+            self.image.write_file("RAMP500K.BIN", None, huge)
         assert len(self.image.files) == self.num_files_in_sample + 1
 
     def test_delete(self):
@@ -118,7 +117,7 @@ class BaseFilesystemModifyTest(object):
         self.image.delete_file(filenames[8])
         assert len(self.image.files) == self.num_files_in_sample + 7
 
-        filename = self.check_entries(entries2, b"SECOND", save="test_delete.atr")
+        filename = self.check_entries(entries2, "SECOND", save="test_delete.atr")
         assert len(self.image.files) == self.num_files_in_sample + 9
 
     def test_delete_all(self):

test/test_ataridos.py

@@ -2,10 +2,11 @@ from __future__ import print_function
 from builtins import object
 from mock import *
 
-from atrcopy import SegmentData, AtariDosFile, InvalidBinaryFile, DefaultSegment, XexContainerSegment
+from atrcopy import SegmentData, AtariDosFile, DefaultSegment, XexContainerSegment, errors
 
 
-class TestAtariDosFile(object):
+
+class TestAtariDosFile:
     def setup(self):
         pass
 
@@ -41,7 +42,7 @@ class TestAtariDosFile(object):
         bytes = [0xff, 0xff, 0x00, 0x60, 0x00, 0x00, 1, 2]
         rawdata = SegmentData(bytes)
         image = AtariDosFile(rawdata)
-        with pytest.raises(InvalidBinaryFile):
+        with pytest.raises(errors.InvalidBinaryFile):
             image.parse_segments()
 
 

test/test_cart.py

@@ -3,10 +3,11 @@ from __future__ import division
 from builtins import object
 from mock import *
 
-from atrcopy import AtariCartImage, SegmentData, InvalidDiskImage
+from atrcopy import AtariCartImage, SegmentData, RomImage, errors
+from atrcopy.cartridge import known_cart_types
 
 
-class TestAtariCart(object):
+class TestAtariCart:
     def setup(self):
         pass
 
@@ -16,26 +17,24 @@ class TestAtariCart(object):
         data[4:8].view(">u4")[0] = cart_type
         return data
 
-    def test_unbanked(self):
+    @pytest.mark.parametrize("k_size,cart_type", [
-        carts = [
             (8, 1),
             (16, 2),
             (8, 21),
             (2, 57),
             (4, 58),
             (4, 59),
-        ]
+        ])
-        for k_size, cart_type in carts:
+    def test_unbanked(self, k_size, cart_type):
-            data = self.get_cart(k_size, cart_type)
+        data = self.get_cart(k_size, cart_type)
-            rawdata = SegmentData(data)
+        rawdata = SegmentData(data)
-            image = AtariCartImage(rawdata, cart_type)
+        image = AtariCartImage(rawdata, cart_type)
-            image.parse_segments()
+        image.parse_segments()
-            assert len(image.segments) == 2
+        assert len(image.segments) == 2
-            assert len(image.segments[0]) == 16
+        assert len(image.segments[0]) == 16
-            assert len(image.segments[1]) == k_size * 1024
+        assert len(image.segments[1]) == k_size * 1024
 
-    def test_banked(self):
+    @pytest.mark.parametrize("k_size,main_size,banked_size,cart_type", [
-        carts = [
             (32, 8, 8, 12),
             (64, 8, 8, 13),
             (64, 8, 8, 67),
@@ -43,16 +42,16 @@ class TestAtariCart(object):
             (256, 8, 8, 23),
             (512, 8, 8, 24),
             (1024, 8, 8, 25),
-        ]
+        ])
-        for k_size, main_size, banked_size, cart_type in carts:
+    def test_banked(self, k_size, main_size, banked_size, cart_type):
-            data = self.get_cart(k_size, cart_type)
+        data = self.get_cart(k_size, cart_type)
-            rawdata = SegmentData(data)
+        rawdata = SegmentData(data)
-            image = AtariCartImage(rawdata, cart_type)
+        image = AtariCartImage(rawdata, cart_type)
-            image.parse_segments()
+        image.parse_segments()
-            assert len(image.segments) == 1 + 1 + (k_size - main_size) //banked_size
+        assert len(image.segments) == 1 + 1 + (k_size - main_size) //banked_size
-            assert len(image.segments[0]) == 16
+        assert len(image.segments[0]) == 16
-            assert len(image.segments[1]) == main_size * 1024
+        assert len(image.segments[1]) == main_size * 1024
-            assert len(image.segments[2]) == banked_size * 1024
+        assert len(image.segments[2]) == banked_size * 1024
 
     def test_bad(self):
         k_size = 32
@@ -60,22 +59,68 @@ class TestAtariCart(object):
         # check for error because invalid data in cart image itself
         data = self.get_cart(k_size, 1337)
         rawdata = SegmentData(data)
-        with pytest.raises(InvalidDiskImage):
+        with pytest.raises(errors.InvalidDiskImage):
             image = AtariCartImage(rawdata, 1337)
-        with pytest.raises(InvalidDiskImage):
+        with pytest.raises(errors.InvalidDiskImage):
             image = AtariCartImage(rawdata, 12)
 
         # check for error with valid cart image, but invalid cart type supplied
         # to the image parser
         data = self.get_cart(k_size, 12)
         rawdata = SegmentData(data)
-        with pytest.raises(InvalidDiskImage):
+        with pytest.raises(errors.InvalidDiskImage):
             image = AtariCartImage(rawdata, 1337)
 
 
+class TestRomCart:
+    def setup(self):
+        pass
+
+    def get_rom(self, k_size):
+        data = np.zeros((k_size * 1024), dtype=np.uint8)
+        return data
+
+    @pytest.mark.parametrize("k_size", [1, 2, 4, 8, 16, 32, 64])
+    def test_typical_rom_sizes(self, k_size):
+        data = self.get_rom(k_size)
+        rawdata = SegmentData(data)
+        rom_image = RomImage(rawdata)
+        rom_image.strict_check()
+        rom_image.parse_segments()
+        assert len(rom_image.segments) == 1
+        assert len(rom_image.segments[0]) == k_size * 1024
+
+    @pytest.mark.parametrize("k_size", [1, 2, 4, 8, 16, 32, 64])
+    def test_invalid_rom_sizes(self, k_size):
+        data = np.zeros((k_size * 1024) + 17, dtype=np.uint8)
+        rawdata = SegmentData(data)
+        with pytest.raises(errors.InvalidDiskImage):
+            rom_image = RomImage(rawdata)
+
+    @pytest.mark.parametrize("cart", known_cart_types)
+    def test_conversion_to_atari_cart(self, cart):
+        cart_type = cart[0]
+        name = cart[1]
+        k_size = cart[2]
+        if "Bounty" in name:
+            return
+        data = self.get_rom(k_size)
+        rawdata = SegmentData(data)
+        rom_image = RomImage(rawdata)
+        rom_image.strict_check()
+        rom_image.parse_segments()
+        new_cart_image = AtariCartImage(rawdata, cart_type)
+        new_cart_image.relaxed_check()
+        new_cart_image.parse_segments()
+        assert new_cart_image.header.valid
+        s = new_cart_image.create_emulator_boot_segment()
+        assert len(s) == len(rawdata) + new_cart_image.header.nominal_length
+        assert s[0:4].tobytes() == b'CART'
+        assert s[4:8].view(dtype=">u4") == cart_type
 
 
 if __name__ == "__main__":
+    from atrcopy.parsers import mime_parse_order
     print("\n".join(mime_parse_order))
 
     t = TestAtariCart()

test/test_container.py

@@ -0,0 +1,41 @@
+from __future__ import print_function
+from builtins import object
+import numpy as np
+
+from mock import *
+
+from atrcopy import SegmentData, iter_parsers
+from atrcopy import errors
+
+
+class BaseContainerTest:
+    base_path = None
+    expected_mime = ""
+
+    @pytest.mark.parametrize("ext", ['.gz', '.bz2', '.xz', '.dcm'])
+    def test_container(self, ext):
+        pathname = self.base_path + ext
+        try:
+            sample_data = np.fromfile(pathname, dtype=np.uint8)
+        except OSError:
+            pass
+        else:
+            rawdata = SegmentData(sample_data.copy())
+            mime, parser = iter_parsers(rawdata)
+            assert mime == self.expected_mime
+            assert len(parser.image.files) == self.num_files_in_sample
+
+class TestContainerAtariDosSDImage(BaseContainerTest):
+    base_path = "../test_data/container_dos_sd_test1.atr"
+    expected_mime = "application/vnd.atari8bit.atr"
+    num_files_in_sample = 5
+
+class TestContainerAtariDosEDImage(BaseContainerTest):
+    base_path = "../test_data/container_dos_ed_test1.atr"
+    expected_mime = "application/vnd.atari8bit.atr"
+    num_files_in_sample = 5
+
+class TestContainerAtariDosDDImage(BaseContainerTest):
+    base_path = "../test_data/container_dos_dd_test1.atr"
+    expected_mime = "application/vnd.atari8bit.atr"
+    num_files_in_sample = 5

test/test_create.py

@@ -4,8 +4,8 @@ import numpy as np
 
 from mock import *
 
-from atrcopy import SegmentData, AtariDosDiskImage, Dos33DiskImage,InvalidBinaryFile, DefaultSegment
+from atrcopy import SegmentData, AtariDosDiskImage, Dos33DiskImage, DefaultSegment
-from atrcopy.errors import *
+from atrcopy import errors
 
 
 def get_image(file_name, diskimage_type):
@@ -15,7 +15,7 @@ def get_image(file_name, diskimage_type):
     return image
 
 
-class BaseCreateTest(object):
+class BaseCreateTest:
     diskimage_type = None
 
     def get_exe_segments(self):
@@ -39,11 +39,11 @@ class BaseCreateTest(object):
         image = get_image(sample_file, diskimage_type)
         segments = self.get_exe_segments()
         try:
-            _ = issubclass(AtrError, expected)
+            _ = issubclass(errors.AtrError, expected)
-            with pytest.raises(InvalidBinaryFile) as e:
+            with pytest.raises(errors.InvalidBinaryFile) as e:
-                file_data, filetype = image.create_executable_file_image(segments, run_addr)
+                file_data, filetype = image.create_executable_file_image(sample_file, segments, run_addr)
         except TypeError:
-            file_data, filetype = image.create_executable_file_image(segments, run_addr)
+            file_data, filetype = image.create_executable_file_image(sample_file, segments, run_addr)
             print(image)
             print(file_data, filetype)
             assert len(file_data) == expected
@@ -53,11 +53,11 @@ class TestAtariDosSDImage(BaseCreateTest):
     diskimage_type = AtariDosDiskImage
 
     @pytest.mark.parametrize("run_addr,expected", [
-        (0x2000, InvalidBinaryFile),
+        (0x2000, errors.InvalidBinaryFile),
         (None, (2 + 6 + (4 + 0x1000) + (4 + 0x1000))),
         (0x4000, (2 + 6 + (4 + 0x1000) + (4 + 0x1000))),
         (0x8000, (2 + 6 + (4 + 0x1000) + (4 + 0x1000))),
-        (0xffff, InvalidBinaryFile),
+        (0xffff, errors.InvalidBinaryFile),
         ])
     def test_exe(self, run_addr, expected, sample_file):
         self.check_exe(sample_file, self.diskimage_type, run_addr, expected)
@@ -68,11 +68,11 @@ class TestDos33Image(BaseCreateTest):
     diskimage_type = Dos33DiskImage
 
     @pytest.mark.parametrize("run_addr,expected", [
-        (0x2000, InvalidBinaryFile),
+        (0x2000, errors.InvalidBinaryFile),
         (None, (4 + (0x9000 - 0x4000))),
         (0x4000, (4 + (0x9000 - 0x4000))),
         (0x8000, (4 + 3 + (0x9000 - 0x4000))),
-        (0xffff, InvalidBinaryFile),
+        (0xffff, errors.InvalidBinaryFile),
         ])
     def test_exe(self, run_addr, expected, sample_file):
         self.check_exe(sample_file, self.diskimage_type, run_addr, expected)

test/test_jsonpickle.py

@@ -12,7 +12,7 @@ import numpy as np
 from atrcopy import DefaultSegment, SegmentData
 
 
-class TestJsonPickle(object):
+class TestJsonPickle:
     def setup(self):
         data = np.arange(2048, dtype=np.uint8)
         self.segment = DefaultSegment(SegmentData(data))

test/test_kboot.py

@@ -10,7 +10,7 @@ from mock import *
 from atrcopy import SegmentData, KBootImage, add_xexboot_header, add_atr_header
 
 
-class TestKbootHeader(object):
+class TestKbootHeader:
     def setup(self):
         pass
 

test/test_segment.py

@@ -8,17 +8,17 @@ import numpy as np
 import pytest
 
 from atrcopy import DefaultSegment, SegmentData, get_xex, interleave_segments, user_bit_mask, diff_bit_mask
-from atrcopy.errors import *
+from atrcopy import errors
 from functools import reduce
 
 
 def get_indexed(segment, num, scale):
     indexes = np.arange(num) * scale
     raw = segment.rawdata.get_indexed(indexes)
-    s = DefaultSegment(raw, segment.start_addr + indexes[0])
+    s = DefaultSegment(raw, segment.origin + indexes[0])
     return s, indexes
 
-class TestSegment1(object):
+class TestSegment1:
     def setup(self):
         self.segments = []
         for i in range(8):
@@ -38,7 +38,7 @@ class TestSegment1(object):
             s[1].set_comment_at(10, "comment 10")
             s[1].set_comment_at(100, "comment 100")
             print(list(s[1].iter_comments_in_segment()))
-            with pytest.raises(InvalidBinaryFile):
+            with pytest.raises(errors.InvalidBinaryFile):
                 seg, subseg = get_xex(s, 0xbeef)
             seg, subseg = get_xex(s)
             assert tuple(seg.data[0:2]) == (0xff, 0xff)
@@ -68,7 +68,7 @@ class TestSegment1(object):
             assert not np.all((c.data[:] - s.data[:]) == 0)
 
 
-class TestIndexed(object):
+class TestIndexed:
     def setup(self):
         data = np.arange(4096, dtype=np.uint8)
         data[1::2] = np.repeat(np.arange(16, dtype=np.uint8), 128)
@@ -97,7 +97,7 @@ class TestIndexed(object):
         assert not sub.rawdata.is_indexed
         for i in range(len(sub)):
             ri = sub.get_raw_index(i)
-            assert ri == sub.start_addr + i
+            assert ri == sub.origin + i
             assert sub[i] == base[ri]
         start, end = sub.byte_bounds_offset()
         assert start == 512
@@ -116,7 +116,7 @@ class TestIndexed(object):
         for i in range(len(indexes)):
             ri = s.get_raw_index(i)
             print(ri, "base[ri]=%d" % base[ri], i, indexes[i], "s[i]=%d" % s[i])
-            assert ri == sub.start_addr + indexes[i]
+            assert ri == sub.origin + indexes[i]
             assert s[i] == base[ri]
         start, end = s.byte_bounds_offset()
         assert start == 0
@@ -126,7 +126,7 @@ class TestIndexed(object):
         s2, indexes2 = get_indexed(s, 64, 3)
         assert s2.rawdata.is_indexed
         for i in range(len(indexes2)):
-            assert s2.get_raw_index(i) == sub.start_addr + indexes2[i] * 3
+            assert s2.get_raw_index(i) == sub.origin + indexes2[i] * 3
         start, end = s.byte_bounds_offset()
         assert start == 0
         assert end == len(base)
@@ -242,7 +242,7 @@ class TestIndexed(object):
         assert not np.all((c.data[:] - s.data[:]) == 0)
 
 
-class TestComments(object):
+class TestComments:
     def setup(self):
         data = np.ones([4000], dtype=np.uint8)
         r = SegmentData(data)
@@ -392,7 +392,7 @@ class TestComments(object):
             assert set(item1[3].values()) == set(item2[3].values())
 
 
-class TestResize(object):
+class TestResize:
     def setup(self):
         data = np.arange(4096, dtype=np.uint8)
         data[1::2] = np.repeat(np.arange(16, dtype=np.uint8), 128)

test/test_serialize.py

@@ -10,7 +10,7 @@ import pytest
 from atrcopy import DefaultSegment, SegmentData, get_xex, interleave_segments
 
 
-class TestSegment(object):
+class TestSegment:
     def setup(self):
         data = np.ones([4000], dtype=np.uint8)
         r = SegmentData(data)
@@ -47,15 +47,15 @@ class TestSegment(object):
         s.set_user_data([r], 4, 99)
 
         out = dict()
-        s.serialize_extra_to_dict(out)
+        s.serialize_session(out)
         print("saved", out)
 
         data = np.ones([4000], dtype=np.uint8)
         r = SegmentData(data)
         s2 = DefaultSegment(r, 0)
-        s2.restore_extra_from_dict(out)
+        s2.restore_session(out)
         out2 = dict()
-        s2.serialize_extra_to_dict(out2)
+        s2.serialize_session(out2)
         print("loaded", out2)
         assert out == out2
 

test/test_utils.py

@@ -3,7 +3,7 @@ from mock import *
 from atrcopy import utils
 
 
-class TestTextToInt(object):
+class TestTextToInt:
     def setup(self):
         pass