Copy documentation into the config definitions.

Makefile

@@ -178,7 +178,7 @@ $(OBJDIR)/$1$3.scp.encodedecode: scripts/encodedecodetest.sh $(FLUXENGINE_BIN) $
 
 endef
 
-$(call do-encodedecodetest,agat840)
+$(call do-encodedecodetest,agat)
 $(call do-encodedecodetest,amiga)
 $(call do-encodedecodetest,apple2,,--140)
 $(call do-encodedecodetest,atarist,,--360)
@@ -189,13 +189,13 @@ $(call do-encodedecodetest,atarist,,--720)
 $(call do-encodedecodetest,atarist,,--740)
 $(call do-encodedecodetest,atarist,,--800)
 $(call do-encodedecodetest,atarist,,--820)
-$(call do-encodedecodetest,bk800)
+$(call do-encodedecodetest,bk)
 $(call do-encodedecodetest,brother,,--120)
 $(call do-encodedecodetest,brother,,--240)
-$(call do-encodedecodetest,commodore1541,scripts/commodore1541_test.textpb,--171)
+$(call do-encodedecodetest,commodore,scripts/commodore1541_test.textpb,--171)
-$(call do-encodedecodetest,commodore1541,scripts/commodore1541_test.textpb,--192)
+$(call do-encodedecodetest,commodore,scripts/commodore1541_test.textpb,--192)
-$(call do-encodedecodetest,commodore1581)
+$(call do-encodedecodetest,commodore,,--800)
-$(call do-encodedecodetest,cmd_fd2000)
+$(call do-encodedecodetest,commodore,,--1620)
 $(call do-encodedecodetest,hplif,,--264)
 $(call do-encodedecodetest,hplif,,--616)
 $(call do-encodedecodetest,hplif,,--770)
@@ -235,7 +235,7 @@ $(call do-corpustest,amiga.flux,amiga.adf,amiga)
 $(call do-corpustest,atarist360.flux,atarist360.st,atarist --360)
 $(call do-corpustest,atarist720.flux,atarist720.st,atarist --720)
 $(call do-corpustest,brother120.flux,brother120.img,brother --120)
-$(call do-corpustest,cmd-fd2000.flux,cmd-fd2000.img,cmd_fd2000)
+$(call do-corpustest,cmd-fd2000.flux,cmd-fd2000.img,commodore --1620)
 $(call do-corpustest,ibm1232.flux,ibm1232.img,ibm --1232)
 $(call do-corpustest,ibm1440.flux,ibm1440.img,ibm --1440)
 $(call do-corpustest,mac800.flux,mac800.dsk,mac --800)

lib/config.proto

@@ -12,12 +12,13 @@ import "lib/drive.proto";
 import "lib/common.proto";
 import "lib/layout.proto";
 
-// NEXT_TAG: 23
+// NEXT_TAG: 24
 message ConfigProto
 {
     optional string comment = 8;
     optional bool is_extension = 13;
     repeated string include = 19;
+	repeated string documentation = 23;
 
     optional LayoutProto layout = 18;
 

scripts/protoencode.cc

@@ -2,12 +2,72 @@
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <fstream>
-#include "fmt/format.h"
+#include <fmt/format.h>
 #include "tests/testproto.pb.h"
 #include "lib/config.pb.h"
+#include <sstream>
+#include <locale>
 
 #define STRINGIFY(s) #s
 
+static uint32_t readu8(std::string::iterator& it, std::string::iterator end)
+{
+	int len;
+    uint32_t c = *it++;
+    if (c < 0x80)
+    {
+        /* Do nothing! */
+		len = 0;
+    }
+    else if (c < 0xc0)
+    {
+        /* Invalid character */
+        c = -1;
+		len = 0;
+    }
+    else if (c < 0xe0)
+    {
+        /* One trailing byte */
+        c &= 0x1f;
+		len = 1;
+    }
+    else if (c < 0xf0)
+    {
+        /* Two trailing bytes */
+        c &= 0x0f;
+		len = 2;
+    }
+    else if (c < 0xf8)
+    {
+        /* Three trailing bytes */
+        c &= 0x07;
+		len = 3;
+    }
+    else if (c < 0xfc)
+    {
+        /* Four trailing bytes */
+        c &= 0x03;
+		len = 4;
+    }
+    else
+    {
+        /* Five trailing bytes */
+        c &= 0x01;
+		len = 5;
+    }
+
+	while (len)
+	{
+		if (it == end)
+			break;
+
+		uint8_t d = *it++;
+		c <<= 6;
+		c += d & 0x3f;
+	}
+    return c;
+}
+
 int main(int argc, const char* argv[])
 {
 	PROTO message;
@@ -19,10 +79,37 @@ int main(int argc, const char* argv[])
 		exit(1);
 	}
 
-    google::protobuf::io::IstreamInputStream istream(&input);
+	std::stringstream ss;
-    if (!google::protobuf::TextFormat::Parse(&istream, &message))
+	std::string s;
+	while (std::getline(input, s, '\n'))
+	{
+		if (s == "<<<")
+		{
+			while (std::getline(input, s, '\n'))
+			{
+				if (s == ">>>")
+					break;
+
+				ss << '"';
+				auto it = s.begin();
+				for (;;)
+				{
+					uint32_t u = readu8(it, s.end());
+					if (!u)
+						break;
+
+					ss << fmt::format("\\u000{:02x}", u);
+				}
+				ss << "\\n\"\n";
+			}
+		}
+		else
+			ss << s << '\n';
+	}
+
+    if (!google::protobuf::TextFormat::ParseFromString(ss.str(), &message))
     {
-        fprintf(stderr, "cannot parse text proto");
+        fprintf(stderr, "cannot parse text proto: %s\n", argv[1]);
         exit(1);
     }
 

src/formats/acornadfs.textpb

@@ -1,5 +1,23 @@
 comment: 'Acorn ADFS family (ro)'
 
+documentation:
+<<<
+Acorn ADFS disks are used by the 6502-based BBC Micro and ARM-based Archimedes
+series of computers. They are yet another variation on MFM encoded IBM scheme
+disks, although with different sector sizes and with the 0-based sector
+identifiers rather than 1-based sector identifiers. The index hole is ignored
+and sectors are written whereever, requiring FluxEngine to do two revolutions
+to read a disk.
+
+There are various different kinds, which should all work out of the box.
+
+Be aware that Acorn logical block numbering goes all the way up side 0 and
+then all the way up side 1. However, FluxEngine uses traditional disk images
+with alternating sides, with the blocks from track 0 side 0 then track 0 side
+1 then track 1 side 0 etc. Most Acorn emulators will use both formats, but
+they might require nudging as the side order can't be reliably autodetected.
+>>>
+
 image_writer {
 	filename: "acornadfs.img"
 	type: IMG
@@ -118,4 +136,4 @@ option_group {
 	}
 }
 
-			
+

src/formats/acorndfs.textpb

@@ -1,5 +1,27 @@
 comment: 'Acorn DFS fmaily'
 
+documentation:
+<<<
+Acorn DFS disks are used by the Acorn Atom and BBC Micro series of computers.
+They are pretty standard FM encoded IBM scheme disks, with 256-sectors and
+0-based sector identifiers. There's nothing particularly special here.
+
+DFS disks are all single-sided, but allow the other side of the disk to be
+used as another volume.
+
+They come in two varieties, 40 track and 80 track. These should both work.
+Some rare disks are both at the same time. FluxEngine can read these but it
+requires a bit of fiddling as they have the same tracks on twice.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The Acorn DFS disc format](https://beebwiki.mdfs.net/Acorn_DFS_disc_format)
+>>>
+
 image_reader {
 	filename: "acorndfs.img"
 	type: IMG
@@ -74,4 +96,3 @@ option_group {
 		}
 	}
 }
-

src/formats/aeslanier.textpb

@@ -1,5 +1,45 @@
 comment: 'AES Lanier "No Problem" 616kB 5.25" 77-track SSDD hard sectored (ro)'
 
+documentation:
+<<<
+Back in 1980 Lanier released a series of very early integrated word processor
+appliances, the No Problem. These were actually [rebranded AES Data Superplus
+machines](http://vintagecomputers.site90.net/aes/). They were gigantic,
+weighed 40kg, and one example I've found cost £13,000 in 1981 (the equivalent
+of nearly £50,000 in 2018!).
+
+8080 machines with 32kB of RAM, they ran their own proprietary word
+processing software off twin 5.25" drive units, but apparently other software
+was available.
+
+The disk format is exceptionally weird. They used 77 track, 32 sector, single-sided
+_hard_ sectored disks, where there were multiple index holes,
+indicating to the hardware where the sectors start. The encoding scheme
+itself is [MMFM (aka
+M2FM)](http://www.retrotechnology.com/herbs_stuff/m2fm.html), an early
+attempt at double-density disk encoding which rapidly got obsoleted by the
+simpler MFM --- and the bytes are stored on disk _backwards_. Even aside from
+the encoding, the format on disk was strange; unified sector header/data
+records, so that the sector header (containing the sector and track number)
+is actually inside the user data.
+
+FluxEngine can read these, but I only have a single, fairly poor example of a
+disk image, and I've had to make a lot of guesses as to the sector format
+based on what looks right. If anyone knows _anything_ about these disks,
+[please get in touch](https://github.com/davidgiven/fluxengine/issues/new).
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  * [SA800 Diskette Storage Drive - Theory Of
+    Operations](http://www.hartetechnologies.com/manuals/Shugart/50664-1_SA800_TheorOp_May78.pdf):
+    talks about MMFM a lot, but the Lanier machines didn't use this disk
+    format.
+>>>
+
 image_writer {
 	filename: "aeslanier.img"
 	type: IMG
@@ -20,3 +60,6 @@ layout {
 		}
 	}
 }
+
+tpi: 96
+

src/formats/agat.textpb

@@ -0,0 +1,53 @@
+comment: 'Agat 840kB 5.25" 80-track DS (ro)'
+
+documentation:
+<<<
+The Agat (Russian: Агат) was a series Soviet-era computer, first released about
+1983. These were based around a 6502 and were nominally Apple II-compatible
+although with enough differences to be problematic.
+
+They could use either standard Apple II 140kB disks, or a proprietary 840kb
+MFM-based double-sided format. FluxEngine supports both of these; this profile
+is for the proprietary format. for the Apple II format, use the `apple2`
+profile.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Magazine article on the
+        Agat](https://sudonull.com/post/54185-Is-AGAT-a-bad-copy-of-Apple)
+
+  - [Forum thread with (some) documentation on the
+        format](https://torlus.com/floppy/forum/viewtopic.php?t=1385)
+>>>
+
+image_writer {
+	filename: "agat.img"
+	type: IMG
+}
+
+layout {
+	tracks: 80
+	sides: 2
+	layoutdata {
+		sector_size: 256
+		physical {
+			start_sector: 0
+			count: 21
+		}
+	}
+}
+
+decoder {
+	agat {}
+}
+
+encoder {
+	agat {}
+}
+
+tpi: 96
+

src/formats/agat840.textpb

@@ -1,27 +0,0 @@
-comment: 'Agat 840kB 5.25" 80-track DS (ro)'
-
-image_writer {
-	filename: "agat.img"
-	type: IMG
-}
-
-layout {
-	tracks: 80
-	sides: 2
-	layoutdata {
-		sector_size: 256
-		physical {
-			start_sector: 0
-			count: 21
-		}
-	}
-}
-
-decoder {
-	agat {}
-}
-
-encoder {
-	agat {}
-}
-

src/formats/amiga.textpb

@@ -1,5 +1,32 @@
 comment: 'Amiga 880kB 3.5" DSDD'
 
+documentation:
+<<<
+Amiga disks use MFM, but don't use IBM scheme. Instead, the entire track is
+read and written as a unit, with each sector butting up against the previous
+one. This saves a lot of space which allows the Amiga to not just store 880kB
+on a DD disk, but _also_ allows an extra 16 bytes of metadata per sector.
+
+This metadata is mostly unused, so the default for FluxEngine is to ignore it
+and just use the 512 bytes of main sector data. If you want it, specify a
+528-byte sector size. The metadata will come after the user data.
+
+Bizarrely, the data in each sector is stored with all the odd bits first, and
+then all the even bits. This is tied into the checksum algorithm, which is
+distinctly subpar and not particularly good at detecting errors.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The Amiga Floppy Boot Process and Physical
+    Layout](https://wiki.amigaos.net/wiki/Amiga_Floppy_Boot_Process_and_Physical_Layout)
+
+  - [The Amiga Disk File FAQ](http://lclevy.free.fr/adflib/adf_info.html)
+>>>
+
 image_reader {
 	filename: "amiga.adf"
 	type: IMG
@@ -34,3 +61,37 @@ filesystem {
 	type: AMIGAFFS
 }
 
+tpi: 96
+
+option_group {
+	comment: "Sector size"
+
+	option {
+		name: "without_metadata"
+		comment: "512-byte sectors"
+		set_by_default: true
+
+		config {
+			layout {
+				layoutdata {
+					sector_size: 512
+				}
+			}
+		}
+	}
+
+	option {
+		name: "with_metadata"
+		comment: "528-byte sectors"
+		set_by_default: true
+
+		config {
+			layout {
+				layoutdata {
+					sector_size: 528
+				}
+			}
+		}
+	}
+}
+

src/formats/ampro.textpb

@@ -1,5 +1,46 @@
 comment: 'Ampro 5.25" family'
 
+documentation:
+<<<
+The Ampro Little Board was a very simple and cheap Z80-based computer from
+1984, which ran CP/M. It was, in fact, a single PCB which you could mount
+on the bottom of a 5.25" drive.
+
+[All about the Ampro Little Board](http://oldcomputers.net/ampro-little-board.html)
+
+It stored either 400kB on a double-sided 40-track drive or 800kB on a
+double-sided 80 track drive. The disk format it used was a slightly quirky
+variation of the standard MFM IBM scheme --- sector numbering starts at 17
+rather than 1 (or Acorn's 0). FluxEngine supports this.
+
+FluxEngine has direct filesystem support for these disks, or you can pass the
+disk images into [cpmtools](http://www.moria.de/~michael/cpmtools/):
+
+```
+$ cpmls -f ampdsdd ampro.img
+0:
+-a60014.e
+amprodsk.com
+bitchk.doc
+bitchk.mac
+cpmmac.mac
+dir.com
+himem.doc
+himem.mac
+kaydiag.lbr
+kayinfo.lbr
+...etc...
+```
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The Ampro Little Board](http://oldcomputers.net/ampro-little-board.html)
+>>>
+
 image_writer {
 	filename: "ampro.img"
 	type: IMG

src/formats/apple2.textpb

@@ -1,5 +1,57 @@
 comment: 'Apple II family'
 
+documentation:
+<<<
+Apple II disks are nominally fairly sensible 40-track, single-sided, 256
+bytes-per-sector jobs. However, they come in two varieties: DOS 3.3/ProDOS and
+above, and pre-DOS 3.3. They use different GCR encoding systems, dubbed
+6-and-2 and 5-and-3, and are mutually incompatible (although in some rare
+cases you can mix 6-and-2 and 5-and-3 sectors on the same disk).
+
+The difference is in the drive controller; the 6-and-2 controller is capable
+of a more efficient encoding, and can fit 16 sectors on a track, storing
+140kB on a disk. The 5-and-3 controller can only fit 13, with a mere 114kB.
+
+Both formats use GCR (in different varieties) in a nice, simple grid of
+sectors, unlike the Macintosh. Like the Macintosh, there's a crazy encoding
+scheme applied to the data before it goes down on disk to speed up
+checksumming.
+
+In addition, a lot of the behaviour of the drive was handled in software.
+This means that Apple II disks can do all kinds of weird things, including
+having spiral tracks! Copy protection for the Apple II was even madder than
+on other systems.
+
+FluxEngine can only read well-behaved 6-and-2 disks. It doesn't even try to
+handle the weird stuff.
+
+Apple DOS also applies logical sector remapping on top of the physical sector
+numbering on the disk, and this _varies_ depending on what the disk is for.
+FluxEngine can remap the sectors from physical to logical using modifiers.  If
+you don't specify a remapping modifier, you get the sectors in the order they
+appear on the disk.
+
+If you don't want an image in physical sector order, specify one of the
+filesystem ordering options. These also select the appropriate file system;
+FluxEngine has read-only support for all of these.
+
+In addition, some third-party systems use 80-track double sides drives, with
+the same underlying disk format. The complication here is that the AppleDOS
+filesystem only supports up to 50 tracks, so it needs tweaking to support
+larger disks. It treats the second side of the disk as a completely different
+volume.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Beneath Apple DOS](https://fabiensanglard.net/fd_proxy/prince_of_persia/Beneath%20Apple%20DOS.pdf)
+
+  - [MAME's ap2_dsk.cpp file](https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap2_dsk.cpp)
+>>>
+
 decoder {
 	apple2 {}
 }

src/formats/atarist.textpb

@@ -1,5 +1,32 @@
 comment: 'Atari ST family'
 
+documentation:
+<<<
+Atari ST disks are standard MFM encoded IBM scheme disks without an IAM header.
+Disks are typically formatted 512 bytes per sector with between 9-10 (sometimes
+11!) sectors per track and 80-82 tracks per side.
+
+For some reason, occasionally formatting software will put an extra IDAM record
+with a sector number of 66 on a disk, which can horribly confuse things. The
+Atari profiles below are configured to ignore these.
+
+Be aware that many PC drives (including mine) won't do the 82 track formats. 
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Atari ST Floppy Drive Hardware
+        Information](https://info-coach.fr/atari/hardware/FD-Hard.php) by Jean
+        Louis-Guerin
+
+  - [Atari ST Floppy Drive Software
+        Information](https://info-coach.fr/atari/software/FD-Soft.php) by Jean
+        Louis-Guerin
+>>>
+
 encoder {
     ibm {
         trackdata {

src/formats/bk.textpb

@@ -0,0 +1,57 @@
+comment: 'BK 800kB 5.25"/3.5" 80-track 10-sector DSDD'
+
+documentation:
+<<<
+The BK (an abbreviation for бытовой компьютер --- 'home computer' in Russian)
+is a Soviet era personal computer from Elektronika based on a PDP-11
+single-chip processor. It was the _only_ official, government approved home
+computer in mass production at the time.
+
+It got a floppy interface in 1989 when the 128kB BK-0011 was released. This
+used a relatively normal double-sided IBM scheme format with 80 sectors and ten
+sectors per track, resulting in 800kB disks. The format is, in fact, identical
+to the Atari ST 800kB format. Either 5.25" or 3.5" drives were used depending
+on what was available at the time, with the same format on both.
+>>>
+
+image_reader {
+	filename: "bk800.img"
+	type: IMG
+}
+
+image_writer {
+	filename: "bk800.img"
+	type: IMG
+}
+
+layout {
+	tracks: 80
+	sides: 2
+	layoutdata {
+		sector_size: 512
+		physical {
+			start_sector: 1
+			count: 10
+		}
+	}
+}
+
+encoder {
+	ibm {
+		trackdata {
+			target_rotational_period_ms: 200
+			target_clock_period_us: 4.0
+			emit_iam: false
+			gap0: 80
+			gap2: 22
+			gap3: 34
+		}
+	}
+}
+
+decoder {
+	ibm {}
+}
+
+tpi: 96
+

src/formats/bk800.textpb

@@ -1,41 +0,0 @@
-comment: 'BK 800kB 5.25"/3.5" 80-track 10-sector DSDD'
-
-image_reader {
-	filename: "bk800.img"
-	type: IMG
-}
-
-image_writer {
-	filename: "bk800.img"
-	type: IMG
-}
-
-layout {
-	tracks: 80
-	sides: 2
-	layoutdata {
-		sector_size: 512
-		physical {
-			start_sector: 1
-			count: 10
-		}
-	}
-}
-
-encoder {
-	ibm {
-		trackdata {
-			target_rotational_period_ms: 200
-			target_clock_period_us: 4.0
-			emit_iam: false
-			gap0: 80
-			gap2: 22
-			gap3: 34
-		}
-	}
-}
-
-decoder {
-	ibm {}
-}
-

src/formats/brother.textpb

@@ -1,5 +1,116 @@
 comment: 'Brother GCR family'
 
+documentation:
+<<<
+Brother word processor disks are weird, using custom tooling and chipsets.
+They are completely not PC compatible in every possible way other than the
+size.
+
+Different word processors use different disk formats --- the only ones
+supported by FluxEngine are the 120kB and 240kB 3.5" formats. The default
+options are for the 240kB format. For the 120kB format, which is 40 track, do
+`fluxengine read brother -s :t=1-79x2`.
+
+Apparently about 20% of Brother word processors have alignment issues which
+means that the disks can't be read by FluxEngine (because the tracks on the
+disk don't line up with the position of the head in a PC drive). The word
+processors themselves solved this by microstepping until they found where the
+real track is, but normal PC drives aren't capable of doing this.  Particularly
+with the 120kB disks, you might want to fiddle with the start track (e.g.
+`:t=0-79x2`) to get a clean read. Keep an eye on the bad sector map that's
+dumped at the end of a read. My word processor likes to put logical track 0 on
+physical track 3, which means that logical track 77 is on physical track 80;
+luckily my PC drive can access track 80.
+
+Using FluxEngine to *write* disks isn't a problem, so the
+simplest solution is to use FluxEngine to create a new disk, with the tracks
+aligned properly, and then use a word processor to copy the files you want
+onto it. The new disk can then be read and you can extract the files.
+Obviously this sucks if you don't actually have a word processor, but I can't
+do anything about that.
+
+If you find one of these misaligned disks then *please* [get in
+touch](https://github.com/davidgiven/fluxengine/issues/new); I want to
+investigate.
+>>>
+
+documentation:
+<<<
+Dealing with misaligned disks
+-----------------------------
+
+While FluxEngine can't read misaligned disks directly, Brother word processors
+_can_. If you have access to a compatible word processor, there's a fairly
+simple workaround to allow you to extract the data:
+
+  1. Format a disk using FluxEngine (by simply writing a blank filesystem image
+         to a disk). This will have the correct alignment to work on a PC drive.
+
+  2. Use a word processor to copy the misaligned disk to the newly formatted
+         disk. The machine will happily adjust itself to both sets of alignments.
+
+  3. Use FluxEngine to read the data off the correctly aligned disk.
+
+I realise this is rather unsatisfactory, as the Brother hardware is becoming
+rarer and they cope rather badly with damaged disks, but this is a limitation
+of the hardware of normal PC drives. (It _is_ possible to deliberately misalign
+a drive to make it match up with a bad disk, but this is for experts only --- I
+wouldn't dare.)
+
+Low level format
+----------------
+
+The drive is a single-sided 3.5" drive spinning at not 300 rpm (I don't know
+the precise speed yet but FluxEngine doesn't care). The 240kB disks have 78
+tracks and the 120kB disks have 39.
+
+Each track has 12 256-byte sectors. The drive ignores the index hole so they're
+lined up all anyhow. As FluxEngine can only read from index to index, it
+actually reads two complete revolutions and reassembles the sectors from that.
+
+The underlying encoding is exceptionally weird; they use two different kinds of
+GCR, one kind for the sector header records and a completely different one for
+the data itself. It also has a completely bizarre CRC variant which a genius on
+StackOverflow reverse engineered for me. However, odd though it may be, it does
+seem pretty robust.
+
+See the source code for the GCR tables and CRC routine.
+
+Sectors are about 16.2ms apart on the disk (at 300 rpm). The header and
+data records are 0.694ms apart. (All measured from the beginning of the
+record.) The sector order is 05a3816b4927, which gives a sector skew of 5.
+
+High level format
+-----------------
+
+Once decoded, you end up with a file system image. FluxEngine supports direct
+filesystem access for both kinds of disks.
+
+### 120kB disks
+
+These disks use a proprietary and very simple file system. It's FAT-like
+with an obvious directory and allocation table. It's supported by FluxEngine.
+
+Any files whose names begin with an asterisk (`*`) will be marked as hidden. If
+the file is named `*boot`, then a boot sector will be created which will load
+and run the file at 0x7000 if the machine is started with CODE+Q pressed. So
+far this has only been confirmed to work on a WP-1.
+
+### 240kB disks
+
+Conversely, the 240kB disks turns out to be a completely normal Microsoft FAT
+file system with a media type of 0x58 --- did you know that FAT supports 256
+byte sectors? I didn't --- of the MSX-DOS variety. There's a faint
+possibility that the word processor is based on MSX-DOS, but I haven't
+reverse engineered it to find out.
+
+Standard Linux mtools will access the filesystem image and allow you to move
+files in and out. However, you'll need to change the media type bytes at
+offsets 0x015 and 0x100 from 0x58 to 0xf0 before mtools will touch it. The
+supplied `brother240tool` will do this. Additionally, FluxEngine's own FAT
+file system supports this.
+>>>
+
 image_reader {
 	filename: "brother.img"
 	type: IMG

src/formats/build.mk

@@ -3,17 +3,15 @@ FORMATS = \
 	acornadfs \
 	acorndfs \
 	aeslanier \
-	agat840 \
+	agat \
 	amiga \
 	ampro \
 	apple2_drive \
 	apple2 \
 	atarist \
-	bk800 \
+	bk \
 	brother \
-	commodore1541 \
+	commodore \
-	commodore1581 \
-	cmd_fd2000 \
 	eco1 \
 	epsonpf10 \
 	f85 \
@@ -26,7 +24,7 @@ FORMATS = \
 	mx \
 	n88basic \
 	northstar \
-	psos800 \
+	psos \
 	rolandd20 \
 	rx50 \
 	shugart_drive \

src/formats/cmd_fd2000.textpb

@@ -1,43 +0,0 @@
-comment: 'CMD FD2000 1620kB 3.5" DSHD'
-
-image_reader {
-	filename: "cmd_fd2000.img"
-	type: IMG
-}
-
-image_writer {
-	filename: "cmd_fd2000.img"
-	type: IMG
-}
-
-layout {
-	tracks: 81
-	sides: 2
-	swap_sides: true
-	layoutdata {
-		sector_size: 1024
-		physical {
-			start_sector: 1
-			count: 10
-		}
-	}
-}
-
-encoder {
-	ibm {
-		trackdata {
-			target_rotational_period_ms: 200
-			target_clock_period_us: 2
-			emit_iam: false
-		}
-	}
-}
-
-decoder {
-	ibm {
-		trackdata {
-			ignore_side_byte: true
-		}
-	}
-}
-

src/formats/commodore.textpb

@@ -0,0 +1,258 @@
+comment: 'Commodore 1541 171kB/192kB 5.25" 35/40-track SS GCR'
+
+documentation:
+<<<
+Commodore 64 disks come in two varieties: GCR, which are the overwhelming
+majority; and MFM, only used on the 1571 and 1581. The latter were (as far as I
+can tell) standard IBM PC format disks with a slightly odd sector count.
+
+The GCR disks are much more interesting. They could store 170kB on a
+single-sided disk (although later drives were double-sided), using a proprietary
+encoding and record scheme; like [Apple Macintosh disks](macintosh.md) they
+stored varying numbers of sectors per track to make the most of the physical
+disk area, although unlike them they did it by changing the bitrate rather than
+adjusting the motor speed.
+
+The drives were also intelligent and ran DOS on a CPU inside them. The
+computer itself knew nothing about file systems. You could even upload
+programs onto the drive and run them there, allowing all sorts of custom disk
+formats, although this was mostly used to compensate for the [cripplingly
+slow connection to the
+computer](https://ilesj.wordpress.com/2014/05/14/1541-why-so-complicated/) of
+300 bytes per second (!). (The drive itself could transfer data reasonably
+quickly.)
+
+A standard 1541 disk has 35 tracks of 17 to 21 sectors, each 256 bytes long
+(sometimes 40 tracks).
+
+A standard 1581 disk has 80 tracks and two sides, each with 10 sectors, 512
+bytes long.
+
+A CMD FD2000 disk (a popular third-party Commodore disk drive) 
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Ruud's Commodore Site: 1541](http://www.baltissen.org/newhtm/1541c.htm):
+    documentation on the 1541 disk format.
+>>>
+
+image_reader {
+	filename: "commodore.d64"
+	type: D64
+}
+
+image_writer {
+	filename: "commodore.d64"
+	type: D64
+}
+
+filesystem {
+	type: CBMFS
+}
+
+option_group {
+	comment: "Format family"
+
+	option {
+		name: "171"
+		comment: "171kB 1541, 35-track variant"
+		set_by_default: true
+		
+		config {
+			layout {
+				sides: 1
+				tracks: 35
+				layoutdata {
+					sector_size: 256
+				}
+				layoutdata {
+					track: 0
+					up_to_track: 16
+					physical {
+						start_sector: 0
+						count: 21
+					}
+				}
+				layoutdata {
+					track: 17
+					up_to_track: 23
+					physical {
+						start_sector: 0
+						count: 19
+					}
+				}
+				layoutdata {
+					track: 24
+					up_to_track: 29
+					physical {
+						start_sector: 0
+						count: 18
+					}
+				}
+				layoutdata {
+					track: 30
+					up_to_track: 39
+					physical {
+						start_sector: 0
+						count: 17
+					}
+				}
+			}
+
+			encoder {
+				c64 {}
+			}
+
+			decoder {
+				c64 {}
+			}
+
+			tpi: 48
+		}
+	}
+		
+	option {
+		name: "192"
+		comment: "192kB 1541, 40-track variant"
+		
+		config {
+			layout {
+				sides: 1
+				tracks: 40
+				layoutdata {
+					sector_size: 256
+				}
+				layoutdata {
+					track: 0
+					up_to_track: 16
+					physical {
+						start_sector: 0
+						count: 21
+					}
+				}
+				layoutdata {
+					track: 17
+					up_to_track: 23
+					physical {
+						start_sector: 0
+						count: 19
+					}
+				}
+				layoutdata {
+					track: 24
+					up_to_track: 29
+					physical {
+						start_sector: 0
+						count: 18
+					}
+				}
+				layoutdata {
+					track: 30
+					up_to_track: 39
+					physical {
+						start_sector: 0
+						count: 17
+					}
+				}
+			}
+
+			encoder {
+				c64 {}
+			}
+
+			decoder {
+				c64 {}
+			}
+
+			tpi: 48
+		}
+	}
+
+	option {
+		name: "800"
+		comment: "800kB 1581"
+
+		config {
+			layout {
+				tracks: 80
+				sides: 2
+				swap_sides: true
+				layoutdata {
+					sector_size: 512
+					physical {
+						start_sector: 1
+						count: 10
+					}
+				}
+			}
+
+			encoder {
+				ibm {
+					trackdata {
+						target_rotational_period_ms: 200
+						target_clock_period_us: 4
+						emit_iam: false
+						gap0: 80
+						gap2: 22
+						gap3: 34
+					}
+				}
+			}
+
+			decoder {
+				ibm {
+					trackdata {
+						ignore_side_byte: true
+					}
+				}
+			}
+
+			tpi: 96
+		}
+	}
+
+	option {
+		name: "1620"
+		comment: "1620kB, CMD FD2000"
+
+		config {
+			layout {
+				tracks: 81
+				sides: 2
+				swap_sides: true
+				layoutdata {
+					sector_size: 1024
+					physical {
+						start_sector: 1
+						count: 10
+					}
+				}
+			}
+
+			encoder {
+				ibm {
+					trackdata {
+						target_rotational_period_ms: 200
+						target_clock_period_us: 2
+						emit_iam: false
+					}
+				}
+			}
+
+			decoder {
+				ibm {
+					trackdata {
+						ignore_side_byte: true
+					}
+				}
+			}
+
+			tpi: 96
+		}
+	}
+}
+		

src/formats/commodore1541.textpb

@@ -1,93 +0,0 @@
-comment: 'Commodore 1541 171kB/192kB 5.25" 35/40-track SS GCR'
-
-image_reader {
-	filename: "commodore1541.d64"
-	type: D64
-}
-
-image_writer {
-	filename: "commodore1541.d64"
-	type: D64
-}
-
-layout {
-	sides: 1
-	tracks: 35
-	layoutdata {
-		sector_size: 256
-	}
-	layoutdata {
-		track: 0
-		up_to_track: 16
-		physical {
-			start_sector: 0
-			count: 21
-		}
-	}
-	layoutdata {
-		track: 17
-		up_to_track: 23
-		physical {
-			start_sector: 0
-			count: 19
-		}
-	}
-	layoutdata {
-		track: 24
-		up_to_track: 29
-		physical {
-			start_sector: 0
-			count: 18
-		}
-	}
-	layoutdata {
-		track: 30
-		up_to_track: 39
-		physical {
-			start_sector: 0
-			count: 17
-		}
-	}
-}
-
-encoder {
-	c64 {}
-}
-
-decoder {
-	c64 {}
-}
-
-tpi: 48
-
-filesystem {
-	type: CBMFS
-}
-
-option_group {
-	comment: "Format family"
-
-	option {
-		name: "171"
-		comment: "35-track variant"
-		set_by_default: true
-		
-		config {
-			layout {
-				tracks: 35
-			}
-		}
-	}
-		
-	option {
-		name: "192"
-		comment: "40-track variant"
-		
-		config {
-			layout {
-				tracks: 40
-			}
-		}
-	}
-}
-		

src/formats/commodore1581.textpb

@@ -1,46 +0,0 @@
-comment: 'Commodore 1581 800kB 3.5" DSDD'
-
-image_reader {
-	filename: "commodore1581.d81"
-	type: IMG
-}
-
-image_writer {
-	filename: "commodore1581.d81"
-	type: IMG
-}
-
-layout {
-	tracks: 80
-	sides: 2
-	swap_sides: true
-	layoutdata {
-		sector_size: 512
-		physical {
-			start_sector: 1
-			count: 10
-		}
-	}
-}
-
-encoder {
-	ibm {
-		trackdata {
-			target_rotational_period_ms: 200
-			target_clock_period_us: 4
-			emit_iam: false
-			gap0: 80
-			gap2: 22
-			gap3: 34
-		}
-	}
-}
-
-decoder {
-	ibm {
-		trackdata {
-			ignore_side_byte: true
-		}
-	}
-}
-

src/formats/eco1.textpb

@@ -1,5 +1,39 @@
 comment: 'VDS Eco1 1210kB 77-track mixed format DSHD (ro)'
 
+documentation:
+<<<
+The Eco1 is a Italian CP/M machine produced in 1982. It had 64kB of RAM, in
+later models expandable up to 384kB, and _two_ Z80 processors. One of these was
+used solely for managing the twin 8" drives, each storing 1.2MB, which was
+quite impressive for a CP/M machine in those days. Visually it is best
+described as 'very brown'.
+
+<div style="text-align: center">
+<a href="vds-eco1.jpg"> <img src="vds-eco1.jpg" style="width:80%" alt="A contemporary advert for the Eco1"/></a>
+</div>
+
+Its format is standard IBM scheme, but with an interesting wrinkle: there are
+_three_ different formatting zones on the disk:
+
+  - Track 0 side 0: 26 sectors, 128 bytes per sector (3296 bytes)
+  - Track 0 side 1: 26 sectors, 256 bytes per sector (6656 bytes)
+  - All others: 16 sectors, 512 bytes per sector (8192 bytes)
+
+The standard `read ibm` command will autodetect and read these disks, but due
+to the format confusing the size autodetection the images need postprocessing
+to be useful, so there's a custom profile for the Eco1 which produces sensible
+images.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Apulio Retrocomputing's page on the
+	Eco1](https://www.apuliaretrocomputing.it/wordpress/?p=8976)
+>>>
+
 image_writer {
 	filename: "eco1.img"
 	type: IMG
@@ -52,3 +86,5 @@ filesystem {
 	}
 }
 
+tpi: 96
+

src/formats/epsonpf10.textpb

@@ -1,5 +1,12 @@
 comment: 'Epson PF-10 40-track DS DD (ro)'
 
+documentation:
+<<<
+The Epson PF10 is the disk unit for the Epson Z80 series of 'laptops', running
+CP/M. It uses a single-sided 40-track 3.5" format, which is unusual, but the
+format itself is yet another IBM scheme variant.
+>>>
+
 image_writer {
 	filename: "epsonpf10.img"
 	type: IMG

src/formats/f85.textpb

@@ -1,5 +1,44 @@
 comment: 'Durango F85 461kB 5.25" 77-track SS (ro)'
 
+documentation:
+<<<
+The Durango F85 was an early office computer based around a 5MHz 8085 processor,
+sold in 1977. It had an impressive 64kB of RAM, upgradable to 128kB, and ran
+its own multitasking operating system call DX-85M, as well as CP/M. It had an
+interesting electric-typewriter form factor, with a little monitor sitting on
+the side of it --- in operation you were facing the 14" printer.
+
+It was touted as being portable. Which it was, if you were strong; the story
+is that they had to do an extensive search to find someone capable of lifting
+it for the following photo...
+
+<div style="text-align: center">
+<img src="durangof85.jpg" style="max-width: 60%" alt="A Durango F85, held precariously">
+</div>
+
+...and even then, they had to airbrush out the tendons in her neck from the
+effort!
+
+It used 5.25 soft-sectored disks storing an impressive-for-those-days
+480kBish on a side, using a proprietary 4-in-5 GCR encoding. They used 77
+tracks, 12 sectors and 512 bytes per sector. Later models used double-sided
+disks; I don't have access to an image of one so don't know how they work
+(there's a suspicious looking spare byte in the sector header which could
+store the side). As always, if you have one, please [get in
+touch](https://github.com/davidgiven/fluxengine/issues/new).
+>>>
+
+documentation:
+<<<
+References
+-----------------
+
+There's amazingly little information about these things.
+
+  * [Chuck Guzis' F85 page](http://www.sydex.com/durango/durango.html) with
+    lots of pictures
+>>>
+
 image_writer {
 	filename: "f85.img"
 	type: IMG
@@ -20,3 +59,6 @@ layout {
 		}
 	}
 }
+
+tpi: 96
+

src/formats/fb100.textpb

@@ -1,5 +1,45 @@
 comment: 'Brother FB-100 100kB 3.5" 40-track SS (ro)'
 
+documentation:
+<<<
+The Brother FB-100 is a serial-attached smart floppy drive used by a several
+different machines for mass storage, including the Tandy Model 100 and
+clones, the Husky Hunter 2, and (bizarrely) several knitting machines. It was
+usually rebadged, sometimes with a cheap paper label stuck over the Brother
+logo, but the most common variant appears to be the Tandy Portable Disk Drive
+or TPDD:
+
+<div style="text-align: center">
+<a href="http://www.old-computers.com/museum/computer.asp?c=233&st=1"> <img src="tpdd.jpg" alt="A Tandy Portable Disk Drive"/></a>
+</div>
+
+It's a bit of an oddball: the disk encoding is FM with a very custom record
+scheme: 40-track single-sided 3.5" disks storing 100kB or so each. Each track
+had only _two_ sectors, each 1280 bytes, but with an additional 12 bytes of
+ID data used for filesystem management.
+
+There was also apparently a TPDD-2 which could store twice as much data, but
+I don't have access to one of those disks.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Tandy Portable Disk Drive operations
+	manual](http://www.classiccmp.org/cini/pdf/Tandy/Portable%20Disk%20Drive%20Operation%20Manual.pdf)
+
+  - [Tandy Portable Disk Drive service
+	manual](https://archive.org/details/TandyPortableDiskDriveSoftwareManual26-3808s)
+
+  - [TPDD design notes (including a dump of the
+	ROM)](http://bitchin100.com/wiki/index.php?title=TPDD_Design_Notes)
+
+  - [Knitting machine FB-100
+	resources](http://www.k2g2.org/wiki:brother_fb-100)
+>>>
+
 image_writer {
 	filename: "fb100.img"
 	type: IMG

src/formats/hplif.textpb

@@ -1,5 +1,16 @@
 comment: 'Hewlett-Packard LIF family'
 
+documentation:
+<<<
+LIF, a.k.a. Logical Interchange Format, is a series of formats used by
+Hewlett-Packard across their entire range of computers, from calculators to
+modern servers. It also defines a simple non-hierarchical filesystem which is
+bizarrely _still_ supported by HP-UX systems.
+
+Floppy-disk wise, they're yet more variations of the standard IBM floppy
+encoding scheme.
+>>>
+
 drive {
 	high_density: false
 }

src/formats/ibm.textpb

@@ -1,5 +1,84 @@
 comment: 'Generic PC 3.5"/5.25" family'
 
+documentation:
+<<<
+IBM scheme disks are _the_ most common disk format, ever. They're used by a
+huge variety of different systems, and they come in a huge variety of different
+forms, but they're all fundamentally the same: either FM or MFM, either single-
+or double-sided, with distinct sector header and data records and no sector
+metadata. Systems which use IBM scheme disks include but are not limited to:
+
+  - IBM PCs (naturally)
+  - Atari ST
+  - late era Apple machines
+  - Acorn machines
+  - the TRS-80
+  - late era Commodore machines (the 1571 and so on)
+  - most CP/M machines
+  - NEC PC-88 series
+  - NEC PC-98 series
+  - Sharp X68000
+  - Fujitsu FM Towns
+  - VAX & PDP-11
+  - etc
+
+FluxEngine supports reading these. However, some variants are more peculiar
+than others, and as a result there are specific decoders which set the defaults
+correctly for certain formats (for example: on PC disks the sector numbers
+start from 1, but on Acorn disks they start from 0). The IBM decoder described
+here is the generic one, and is suited for 'conventional' PC disks. While you
+can read all the variant formats with it if you use the right set of arguments,
+it's easier to use the specific decoder.
+
+There is a generic decoder which should adjust automatically to whichever disk
+format you are using, but it's unreliable and not recommended. This format
+should also be used if you are writing images such as DIM which specify the
+image format. FluxEngine will use these parameters.
+>>>
+
+documentation:
+<<<
+Mixed-format disks
+------------------
+
+Some disks, such as those belonging to early CP/M machines, or N88-Basic disks
+(for PC-88 and PC-98), have more than one format on the disk at once. Typically,
+the first few tracks will be low-density FM encoded and will be read by the
+machine's ROM; those tracks contain new floppy drive handling code capable of
+coping with MFM data, and so the rest of the disk will use that, allowing them
+to store more data.
+
+FluxEngine can read these fine, but it tends to get a bit confused when it sees
+tracks with differing numbers of sectors --- if track 0 has 32 sectors but
+track 1 has 16, it will assume that sectors 16..31 are missing on track 1 and
+size the image file accordingly. This can be worked around by specifying the
+size of each track; see the `eco1` read profile for an example.
+
+N88-Basic format floppies can be written by either specifying the `n88basic`
+format, or by using D88 or NFD format images which include explicit sector
+layout information.
+
+Writing other formats can be made to work too, by creating a custom format
+specifier, using the `n88basic` format as an example.
+Please [get in touch](https://github.com/davidgiven/fluxengine/issues/new) if
+you have specific requirements.
+
+360rpm 3.5" disks
+-----------------
+
+Japanese PCs (NEC PC-98, Sharp X68000, Fujitsu FM Towns) spin their floppy
+drives at 360rpm rather than the more typical 300rpm. This was done in order
+to be fully backwards compatible with 5.25" disks, while using the exact
+same floppy controller. Later models of the PC-9821, as well as most USB floppy
+drives, feature "tri-mode" support which in addition to normal 300rpm modes,
+can change their speed to read and write 360rpm DD and HD disks.
+
+Neither the FluxEngine or Greaseweazle hardware can currently command a
+tri-mode drive to spin at 360rpm. However on both devices the FluxEngine
+software is capable of both reading and writing 300rpm disks at 360rpm and vice
+versa, so it shouldn't matter.
+>>>
+
 image_reader {
 	filename: "ibm.img"
 	type: IMG

src/formats/icl30.textpb

@@ -1,4 +1,11 @@
-comment: 'ICL Model 30 263kB 34-track DSSD (ro)'
+comment: 'ICL Model 30 263kB 35-track DSSD'
+
+documentation:
+<<<
+The ICL Model 30 is a reasonably standard CP/M machine using 35-track single
+density disks and the traditional CP/M 128-byte secotrs --- 30 of them per
+track! Other than that it's another IBM scheme variation.
+>>>
 
 image_writer {
 	filename: "icl30.img"

src/formats/mac.textpb

@@ -1,5 +1,62 @@
 comment: 'Macintosh GCR family'
 
+documentation:
+<<<
+Macintosh disks come in two varieties: the newer 1440kB ones, which are
+perfectly ordinary PC disks you should use the `ibm` profile to read them, and
+the older 800kB disks (and 400kB for the single sides ones). They have 80
+tracks and up to 12 sectors per track.
+
+They are also completely insane.
+
+It's not just the weird, custom GCR encoding. It's not just the utterly
+bizarre additional encoding/checksum built on top of that where [every byte
+is mutated according to the previous bytes in the
+sector](https://www.bigmessowires.com/2011/10/02/crazy-disk-encoding-schemes/).
+It's not just the odd way in which disks think they have four sides, two on one
+side and two on the other, so that the track byte stores only the bottom 6 bits
+of the track number. It's not just the way that Macintosh sectors are 524 bytes
+long. No, it's the way the Macintosh drive changes speed depending on which
+track it's looking at, so that each track contains a different amount of data.
+
+The reason for this is actually quite sensible: the tracks towards the centre
+of the disk are obviously moving more slowly, so you can't pack the bits in
+quite as closely (due to limitations in the magnetic media). You can use a
+higher bitrate at the edge of the disk than in the middle. Many platforms, for
+example the Commodore 64 1541 drive, changed bitrate this way.
+
+But Macintosh disks used a constant bitrate and changed the speed that the disk
+spun instead to achieve the same effect...
+
+_Anyway_: FluxEngine will read them fine on conventional drives.  Because it's
+clever.
+
+Macintosh computers never really used the twelve bytes of metadata and the
+standard for disk images is to omit it. If you want them, specify that you want
+524-byte sectors. The metadata will follow the 512 bytes of user data.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [MAME's ap_dsk35.cpp file](https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap_dsk35.cpp),
+    without which I'd never have managed to do this
+
+  - [Crazy Disk Encoding
+    Schemes](https://www.bigmessowires.com/2011/10/02/crazy-disk-encoding-schemes/), which made
+    me realise just how nuts the format is
+
+  - [Les Disquettes et le drive Disk II](http://www.hackzapple.com/DISKII/DISKIITECH.HTM), an
+    epicly detailed writeup of the Apple II disk format (which is closely related)
+
+  - [The DiskCopy 4.2
+        format](https://www.discferret.com/wiki/Apple_DiskCopy_4.2), described on
+        the DiskFerret website.
+>>>
+
+
 image_reader {
 	filename: "mac.dsk"
 	type: IMG

src/formats/micropolis.textpb

@@ -1,5 +1,64 @@
 comment: ' Micropolis format family'
 
+documentation:
+<<<
+Micropolis MetaFloppy disks use MFM and hard sectors. Mod I was 48 TPI and
+stored 143k per side. Mod II was 100 TPI and stored 315k per side. Each of the
+16 sectors contains 266 bytes of "user data," allowing 10 bytes of metadata for
+use by the operating system. Micropolis DOS (MDOS) used the metadata bytes, but
+CP/M did not.
+
+Some later systems were Micropolis-compatible and so were also 100 TPI, like
+the Vector Graphic Dual-Mode Disk Controller which was paired with a Tandon
+drive.
+
+**Important note:** You _cannot_ read these disks with a normal PC drive, as
+these drives are 96tpi. The track spacing is determined by the physical geometry
+of the drive and can't be changed in software. You'll need to get hold of a
+100tpi Micropolis drive. Luckily these seem to use the same connector and
+pinout as a 96tpi PC 5.25" drive. In use they should be identical.
+
+While most operating systems use the standard Micropolis checksum, Vector
+Graphic MZOS uses a unique checksum.  The decoder will automatically detect
+the checksum type in use; however, a specific checksum type may be forced
+using the `--decoder.micropolis.checksum_type=n` where the type is one of:
+
+| Type | Description                             |
+|------|-----------------------------------------|
+| 0    | Automatically detect                    |
+| 1    | Standard Micropolis (MDOS, CP/M, OASIS) |
+| 2    | Vector Graphic MZOS                     |
+
+The [CP/M BIOS](https://www.seasip.info/Cpm/bios.html) defined SELDSK, SETTRK,
+and SETSEC, but no function to select the head/side. Double-sided floppies
+could be represented as having either twice the number of sectors, for CHS, or
+twice the number of tracks, HCS; the second side's tracks in opposite order
+logically followed the first side (e.g., tracks 77-153). Micropolis disks
+tended to be the latter. FluxEngine always emits CHS format disks, so you may
+need to apply extra options to change the format if desired.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Micropolis 1040/1050 S-100 Floppy Disk Subsystems User's Manual][micropolis1040/1050].
+    Section 6, pages 261-266. Documents pre-ECC sector format. Note that the
+    entire record, starting at the sync byte, is controlled by software
+
+  - [Vector Graphic Dual-Mode Disk Controller Board Engineering Documentation][vectordualmode].
+    Section 1.6.2. Documents ECC sector format
+
+  - [AltairZ80 Simulator Usage Manual][altairz80]. Section 10.6. Documents ECC
+    sector format and VGI file format
+
+[micropolis1040/1050]: http://www.bitsavers.org/pdf/micropolis/metafloppy/1084-01_1040_1050_Users_Manual_Apr79.pdf
+[vectordualmode]: http://bitsavers.org/pdf/vectorGraphic/hardware/7200-1200-02-1_Dual-Mode_Disk_Controller_Board_Engineering_Documentation_Feb81.pdf
+[altairz80]: http://www.bitsavers.org/simh.trailing-edge.com_201206/pdf/altairz80_doc.pdf
+>>>
+
+
 drive {
 	hard_sector_count: 16
 	tpi: 100

src/formats/mx.textpb

@@ -1,5 +1,59 @@
 comment: 'DVK MX family'
 
+documentation:
+<<<
+The DVK (in Russian, ДВК, Диалоговый вычислительный комплекс or Dialogue
+Computing Complex) was a late 1970s Soviet personal computer, a cut-down
+version of the professional SM EVM (СМ ЭВМ, abbreviation of Система Малых ЭВМ
+--- literally System of Mini Computers), which _itself_ was an unlicensed
+clone of the PDP-11. The MX board was an early floppy drive controller board
+for it.
+
+<div style="text-align: center">
+<a href="http://www.leningrad.su/museum/show_big.php?n=1006"><img src="dvk3m.jpg" style="max-width: 60%" alt="A DVK computer"></a>
+</div>
+
+The MX format is interesting in that it has to be read a track at a time. The
+format contains the usual ID prologue at the beginning of the track, then
+eleven data blocks and checksums, then the epilogue, then it stops. The
+actual encoding is normal FM. There were four different disk variants, in all
+combinations of single- and double-sided and 40- and 80-tracked; but every
+track contained eleven 256-byte sectors.
+
+The format varies subtly depending on whether you're using the 'new' driver
+or the 'old' driver. FluxEngine should read both.
+
+A track is:
+
+  * 8 x 0x0000 words (FM encoded as 01010101...)
+  * 1 x 0x00F3 --- start of track
+  * 1 x 0xnnnn --- track number
+  * 11 of:
+    * 128 words (256 bytes) of data
+    * 16 bit checksum
+  * **if 'new' format:**
+    * 3 x 0x83nn --- `n = (track_number<<1) + side_number`
+  * **if 'old' format:**
+    * 3 x 0x8301
+
+The checksum is just the unsigned integer sum of all the words in the sector.
+Words are all stored little-endian.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The Soviet Digital Electronics
+    Museum](http://www.leningrad.su/museum/main.php) (source of the image
+    above)
+
+  - [a random post on the HxC2001 support
+    forum](http://torlus.com/floppy/forum/viewtopic.php?t=1384) with lots of
+    information on the format
+>>>
+
 image_writer {
 	filename: "mx.img"
 	type: IMG

src/formats/n88basic.textpb

@@ -1,5 +1,15 @@
 comment: 'N88-BASIC 5.25"/3.5" 77-track 26-sector DSHD'
 
+documentation:
+<<<
+The N88-BASIC disk format is the one used by the operating system of the same
+name for the Japanese PC8800 and PC98 computers. It is another IBM scheme
+variant, and is very similar to some mixed-format CP/M disk formats, where
+track 0 side 0 uses 128-byte single density sectors and the rest of the disk
+uses 512-byte double density sectors. (The reason for this is that the PC8800
+boot ROM could only read single density data.)
+>>>
+
 drive {
 	high_density: true
 }
@@ -62,3 +72,6 @@ encoder {
 decoder {
 	ibm {}
 }
+
+tpi: 96
+

src/formats/northstar.textpb

@@ -1,5 +1,36 @@
 comment: 'Northstar family'
 
+documentation:
+<<<
+Northstar Floppy disks use 10-sector hard sectored disks with either FM or MFM
+encoding.  They may be single- or double-sided.  Each of the 10 sectors contains
+256 (FM) or 512 (MFM) bytes of data.  The disk has 35 cylinders, with tracks 0-
+34 on side 0, and tracks 35-69 on side 1.  Tracks on side 1 are numbered "back-
+wards" in that track 35 corresponds to cylinder 34, side 1, and track 69
+corresponds to cylinder 0, side 1.
+
+The Northstar sector format does not include any head positioning information.
+As such, reads from Northstar floppies need to by synchronized with the index
+pulse, in order to properly identify the sector being read.  This is handled
+automatically by FluxEngine.
+
+Due to the nature of the track ordering on side 1, an .nsi image reader and
+writer are provided for double-sided disks.  The .nsi image writer supports
+both single- and double-sided disks; however single-sided .nsi images are
+equivalent to .img images.
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [MICRO-DISK SYSTEM MDS-A-D DOUBLE DENSITY Manual][northstar_mds].
+    Page 33 documents sector format for single- and double-density.
+
+[northstar_mds]: http://bitsavers.org/pdf/northstar/boards/Northstar_MDS-A-D_1978.pdf
+>>>
+
 image_reader {
 	filename: "northstar.nsi"
 	type: NSI

src/formats/psos.textpb

@@ -0,0 +1,76 @@
+comment: 'pSOS generic 800kB DSDD with PHILE'
+
+documentation:
+<<<
+pSOS was an influential real-time operating system from the 1980s, used mainly
+on 68000-based machines, lasting up until about 2000 when it was bought (and
+cancelled) by Wind River. It had its own floppy disk format and file system,
+both of which are partially supported here.
+
+The PHILE file system is almost completely undocumented and so many of the data
+structures have had to be reverse engineered and are not well known. Please
+[get in touch](https://github.com/davidgiven/fluxengine/issues/new) if you know
+anything about it.
+
+The floppy disk format itself is an IBM scheme variation with 1024-byte sectors
+and, oddly, swapped sides.
+>>>
+
+drive {
+	high_density: false
+	rotational_period_ms: 200
+}
+
+image_reader {
+	filename: "pme.img"
+	type: IMG
+}
+
+image_writer {
+	filename: "pme.img"
+	type: IMG
+}
+
+layout {
+	tracks: 80
+	sides: 2
+	order: HCS
+	swap_sides: true
+	layoutdata {
+		sector_size: 1024
+		physical {
+			sector: 1
+			sector: 2
+			sector: 3
+			sector: 4
+			sector: 5
+		}
+	}
+}
+
+encoder {
+	ibm {
+		trackdata {
+			target_rotational_period_ms: 200
+			target_clock_period_us: 4
+			gap0: 80
+			gap2: 22
+			gap3: 80
+		}
+	}
+}
+
+decoder {
+	ibm {
+		trackdata {
+			ignore_side_byte: true
+		}
+	}
+}
+
+filesystem {
+	type: PHILE
+}
+	
+tpi: 96
+

src/formats/psos800.textpb

@@ -1,58 +0,0 @@
-comment: 'pSOS generic 800kB DSDD with PHILE'
-
-drive {
-	high_density: false
-	rotational_period_ms: 200
-}
-
-image_reader {
-	filename: "pme.img"
-	type: IMG
-}
-
-image_writer {
-	filename: "pme.img"
-	type: IMG
-}
-
-layout {
-	tracks: 80
-	sides: 2
-	order: HCS
-	swap_sides: true
-	layoutdata {
-		sector_size: 1024
-		physical {
-			sector: 1
-			sector: 2
-			sector: 3
-			sector: 4
-			sector: 5
-		}
-	}
-}
-
-encoder {
-	ibm {
-		trackdata {
-			target_rotational_period_ms: 200
-			target_clock_period_us: 4
-			gap0: 80
-			gap2: 22
-			gap3: 80
-		}
-	}
-}
-
-decoder {
-	ibm {
-		trackdata {
-			ignore_side_byte: true
-		}
-	}
-}
-
-filesystem {
-	type: PHILE
-}
-	

src/formats/rolandd20.textpb

@@ -1,5 +1,21 @@
 comment: 'Roland D20'
 
+documentation:
+<<<
+The Roland D20 is a classic electronic synthesiser with a built-in floppy
+drive, used for saving MIDI sequences and samples.
+
+Weirdly, it seems to use precisely the same format as the Brother word
+processors: a thoroughly non-IBM-compatible custom GCR system.
+
+FluxEngine pretends to support this, but it has had almost no testing, the only
+disk image I have seen for it was mostly corrupt, and very little is known
+about the format, so I have no idea whether it's correct or not.
+
+Please [get in touch](https://github.com/davidgiven/fluxengine/issues/new) if
+you know anything about it.
+>>>
+
 image_writer {
 	filename: "rolandd20.img"
 	type: IMG
@@ -26,3 +42,4 @@ drive {
 	head_bias: 1
 }
 
+tpi: 96

src/formats/rx50.textpb

@@ -1,4 +1,11 @@
-comment: 'Digital RX50 400kB 5.25" 80-track 10-sector SSQD'
+comment: 'Digital RX50 400kB 5.25" 80-track 10-sector SSDD'
+
+documentation:
+<<<
+The Digital RX50 is one of the external floppy drive units used by Digital's
+range of computers, especially the DEC Rainbow microcomputer. It is a fairly
+vanilla single-sided IBM scheme variation.
+>>>
 
 drive {
 	high_density: true
@@ -39,3 +46,5 @@ encoder {
 decoder {
 	ibm {}
 }
+
+tpi: 96

src/formats/smaky6.textpb

@@ -1,5 +1,31 @@
 comment: 'Smaky 6 308kB 5.25" 77-track 16-sector SSDD, hard sectored'
 
+documentation:
+<<<
+The Smaky 6 is a Swiss computer from 1978 produced by Epsitec. It's based
+around a Z80 processor and has one or two Micropolis 5.25" drives which use
+16-sector hard sectored disks. The disk format is single-sided with 77 tracks
+and 256-byte sectors, resulting in 308kB disks. It uses MFM with a custom
+sector record scheme. It was later superceded by a 68000-based Smaky which used
+different disks.
+
+FluxEngine supports these, although because the Micropolis drives use a 100tpi
+track pitch, you can't read Smaky 6 disks with a normal PC 96tpi drive. You
+will have to find a 100tpi drive from somewhere (they're rare).
+
+There is experimental read-only support for the Smaky 6 filesystem, allowing
+the directory to be listed and files read from disks. It's not known whether
+this is completely correct, so don't trust it!
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [Smaky Info, 1978-2002 (in French)](https://www.smaky.ch/theme.php?id=sminfo)
+>>>
+
 image_writer {
 	filename: "smaky6.img"
 	type: IMG
@@ -31,3 +57,5 @@ filesystem {
 	type: SMAKY6
 }
 
+tpi: 100
+

src/formats/tids990.textpb

@@ -1,5 +1,32 @@
 comment: 'Texas Instruments DS990 1126kB 8" DSSD'
 
+documentation:
+<<<
+The Texas Instruments DS990 was a multiuser modular computing system from 1998,
+based around the TMS-9900 processor (as used by the TI-99). It had an 8" floppy
+drive module, the FD1000, which was a 77-track, 288-byte sector FM/MFM system
+with 26 sectors per track. The encoding scheme was very similar to a simplified
+version of the IBM scheme, but of course not compatible. A double-sided disk
+would store a very satisfactory 1126kB of data; here's one at <a
+href="https://www.old-computers.com/museum/computer.asp?st=1&c=1025">old-computers.com</a>:
+
+<div style="text-align: center">
+<a href="https://www.old-computers.com/museum/computer.asp?st=1&c=1025">
+<img src="tids990.jpg" style="max-width: 60%" alt="A DS990 at old-computers.com"></a>
+</div>
+
+FluxEngine will read and write these (but only the DSDD MFM variant).
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The FD1000 Depot Maintenance
+        Manual](http://www.bitsavers.org/pdf/ti/990/disk/2261885-9701_FD1000depotVo1_Jan81.pdf)
+>>>
+
 image_reader {
 	filename: "tids990.img"
 	type: IMG
@@ -53,3 +80,5 @@ encoder {
 decoder {
 	tids990 {}
 }
+
+tpi: 96

src/formats/tiki.textpb

@@ -1,5 +1,13 @@
 comment: 'Tiki 100 family'
 
+documentation:
+<<<
+The Tiki 100 is a Z80-based Norwegian microcomputer from the mid 1980s intended
+for eductional use. It mostly ran an unbranded CP/M clone, and uses fairly
+normal CP/M disks --- IBM scheme and from 128 to 512 bytes per sector depending
+on the precise format. 
+>>>
+
 image_writer {
 	filename: "tiki.img"
 	type: IMG
@@ -47,7 +55,7 @@ option_group {
 
 	option {
 		name: "200"
-		comment: '200kB 40-track 10-sector SSSD'
+		comment: '200kB 40-track 10-sector SSDD'
 
 		config {
 			layout {
@@ -78,7 +86,7 @@ option_group {
 
 	option {
 		name: "400"
-		comment: '400kB 40-track 10-sector DSSD'
+		comment: '400kB 40-track 10-sector DSDD'
 
 		config {
 			layout {
@@ -110,7 +118,7 @@ option_group {
 
 	option {
 		name: "800"
-		comment: '800kB 80-track 10-sector DSSD'
+		comment: '800kB 80-track 10-sector DSDD'
 
 		config {
 			layout {

src/formats/victor9k.textpb

@@ -1,5 +1,51 @@
 comment: 'Victor 9000 / Sirius One family'
 
+documentation:
+<<<
+The Victor 9000 / Sirius One was a rather strange old 8086-based machine
+which used a disk format very reminiscent of the Commodore format; not a
+coincidence, as Chuck Peddle designed them both. They're 80-track, 512-byte
+sector GCR disks, with a variable-speed drive and a varying number of sectors
+per track --- from 19 to 12. Disks can be double-sided, meaning that they can
+store 1224kB per disk, which was almost unheard of back then. Because the way
+that the tracks on head 1 are offset from head 0 (this happens with all disks),
+the speed zone allocation on head 1 differs from head 0...
+
+| Zone | Head 0 tracks | Head 1 tracks | Sectors | Original period (ms) |
+|:----:|:-------------:|:-------------:|:-------:|:--------------------:|
+| 0    | 0-3           |               | 19      | 237.9                |
+| 1    | 4-15          | 0-7           | 18      | 224.5                |
+| 2    | 16-26         | 8-18          | 17      | 212.2                |
+| 3    | 27-37         | 19-29         | 16      | 199.9                |
+| 4    | 38-47\*       | 30-39\*       | 15      | 187.6                |
+| 5    | 48-59         | 40-51         | 14      | 175.3                |
+| 6    | 60-70         | 52-62         | 13      | 163.0                |
+| 7    | 71-79         | 63-74         | 12      | 149.6                |
+| 8    |               | 75-79         | 11      | 144.0                |
+
+(The Original Period column is the original rotation rate. When used in
+FluxEngine, the disk always spins at 360 rpm, which corresponds to a rotational
+period of 166 ms.)
+
+\*The Victor 9000 Hardware Reference Manual has a bug in the documentation 
+and lists Zone 4 as ending with track 48 on head 0 and track 40 on head 1. 
+The above table matches observed data on various disks and the assembly 
+code in the boot loader, which ends Zone 4 with track 47 on head 0 
+and track 39 on Head 1.
+
+FluxEngine can read and write both the single-sided and double-sided variants. 
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  - [The Victor 9000 technical reference manual](http://bitsavers.org/pdf/victor/victor9000/Victor9000TechRef_Jun82.pdf)
+
+  - [DiskFerret's Victor 9000 format guide](https://discferret.com/wiki/Victor_9000_format)
+>>>
+
 image_reader {
 	filename: "victor9k.img"
 	type: IMG

src/formats/zilogmcz.textpb

@@ -1,5 +1,42 @@
 comment: 'Zilog MCZ 320kB 8" 77-track SS hard-sectored (ro)'
 
+documentation:
+<<<
+The Zilog MCZ is an extremely early Z80 development system, produced by
+Zilog, which came out in 1976. It used twin 8-inch hard sectored floppy
+drives; here's one at the <a
+href="http://www.computinghistory.org.uk/det/12157/Zilog-Z-80-Microcomputer-System/">Centre
+for Computing History</a>:
+
+<div style="text-align: center">
+<a href="http://www.computinghistory.org.uk/det/12157/Zilog-Z-80-Microcomputer-System/">
+<img src="zilogmcz.jpg" style="max-width: 60%" alt="A Zilog MCZ at the Centre For Computing History"></a>
+</div>
+
+The MCZ ran Zilog's own operating system, Z80-RIO, and used 77 track
+single-sided disks, with 32 sectors (each marked by an index hole), with 132
+bytes per sector --- 128 bytes of user payload plus two two-byte metadata
+words used to construct linked lists of sectors for storing files. These
+stored 320kB each.
+
+FluxEngine has experimental read support for these disks, based on a single
+Catweasel flux file I've been able to obtain, which only contained 70 tracks.
+I haven't been able to try this for real. If anyone has any of these disks,
+an 8-inch drive, a FluxEngine and the appropriate adapter, please [get in
+touch](https://github.com/davidgiven/fluxengine/issues/new)...
+>>>
+
+documentation:
+<<<
+References
+----------
+
+  * [About the Zilog MCZ](http://www.retrotechnology.com/restore/zilog.html),
+    containing lots of useful links
+
+  * [The hardware user's manual](https://amaus.org/static/S100/zilog/ZDS/Zilog%20ZDS%201-25%20Hardware%20Users%20Manual.pdf)
+>>>
+
 image_writer {
 	filename: "zilogmcz.img"
 	type: IMG
@@ -20,3 +57,6 @@ layout {
 		}
 	}
 }
+
+tpi: 48
+

tests/configs.cc

@@ -29,16 +29,12 @@ struct fmt::formatter<std::vector<std::string>>
         FormatContext& ctx) const -> decltype(ctx.out())
     {
 		auto it = ctx.out();
-		bool first = true;
 		for (const auto& s : vector)
 		{
 			if (s.empty())
 				continue;
 
-			if (!first)
+			it = fmt::format_to(it, "+{}", s);
-				it = fmt::format_to(it, "+");
-			it = fmt::format_to(it, "{}", s);
-			first = false;
 		}
 
         return it;
@@ -89,7 +85,7 @@ static void validateConfigWithOptions(std::string baseConfigName,
     /* All configs must have a tpi. */
 
     if (!config.has_tpi())
-        error("{}+{}: no tpi set", baseConfigName, options);
+        error("{}{}: no tpi set", baseConfigName, options);
 }
 
 static void validateToplevelConfig(std::string name)
@@ -149,17 +145,20 @@ static void validateToplevelConfig(std::string name)
     /* For each permutation of options, verify the complete config. */
 
     auto combinations = generateCombinations(optionGroups);
-    for (const auto& group : combinations)
+	if (combinations.empty())
-    {
+		validateConfigWithOptions(name, {}, config);
-        ConfigProto configWithOption = config;
+	else
-        for (const auto& optionName : group)
+		for (const auto& group : combinations)
-        {
+		{
-            if (!optionName.empty())
+			ConfigProto configWithOption = config;
-                configWithOption.MergeFrom(
+			for (const auto& optionName : group)
-                    optionProtos.at(optionName)->config());
+			{
-        }
+				if (!optionName.empty())
-        validateConfigWithOptions(name, group, configWithOption);
+					configWithOption.MergeFrom(
-    }
+						optionProtos.at(optionName)->config());
+			}
+			validateConfigWithOptions(name, group, configWithOption);
+		}
 }
 
 int main(int argc, const char* argv[])