Merge pull request #463 from hharte/improve_micropolis_decoder

Improve micropolis decoder

arch/micropolis/decoder.cc

@@ -8,10 +8,18 @@
 #include "fmt/format.h"
 #include "lib/decoders/decoders.pb.h"
 
-/* The sector has a preamble of MFM 0x00s and uses 0xFF as a sync pattern. */
-static const FluxPattern SECTOR_SYNC_PATTERN(32, 0xaaaa5555);
+/* The sector has a preamble of MFM 0x00s and uses 0xFF as a sync pattern.
+ *
+ * 00        00        00        F         F
+ * 0000 0000 0000 0000 0000 0000 0101 0101 0101 0101
+ * A    A    A    A    A    A    5    5    5    5
+ */
+static const FluxPattern SECTOR_SYNC_PATTERN(64, 0xAAAAAAAAAAAA5555LL);
 
-/* Adds all bytes, with carry. */
+/* Pattern to skip past current SYNC. */
+static const FluxPattern SECTOR_ADVANCE_PATTERN(64, 0xAAAAAAAAAAAAAAAALL);
+
+/* Standard Micropolis checksum.  Adds all bytes, with carry. */
 uint8_t micropolisChecksum(const Bytes& bytes) {
 	ByteReader br(bytes);
 	uint16_t sum = 0;
@@ -25,43 +33,129 @@ uint8_t micropolisChecksum(const Bytes& bytes) {
 	return sum & 0xFF;
 }
 
+/* Vector MZOS does not use the standard Micropolis checksum.
+ * The checksum is initially 0.
+ * For each data byte in the 256-byte payload, rotate left,
+ * carrying bit 7 to bit 0.  XOR with the current checksum.
+ *
+ * Unlike the Micropolis checksum, this does not cover the 12-byte
+ * header (track, sector, 10 OS-specific bytes.)
+ */
+uint8_t mzosChecksum(const Bytes& bytes) {
+	ByteReader br(bytes);
+	uint8_t checksum = 0;
+	uint8_t databyte;
+
+	while (!br.eof()) {
+		databyte = br.read_8();
+	checksum ^= ((databyte << 1) | (databyte >> 7));
+	}
+
+	return checksum;
+}
+
 class MicropolisDecoder : public AbstractDecoder
 {
 public:
 	MicropolisDecoder(const DecoderProto& config):
 		AbstractDecoder(config),
 		_config(config.micropolis())
-	{}
+	{
+		_checksumType = _config.checksum_type();
+	}
 
 	nanoseconds_t advanceToNextRecord() override
 	{
-		seekToIndexMark();
-		return seekToPattern(SECTOR_SYNC_PATTERN);
+		nanoseconds_t now = tell().ns();
+
+		/* For all but the first sector, seek to the next sector pulse.
+		 * The first sector does not contain the sector pulse in the fluxmap.
+		 */
+		if (now != 0) {
+			seekToIndexMark();
+			now = tell().ns();
+		}
+
+		/* Discard a possible partial sector at the end of the track.
+		 * This partial sector could be mistaken for a conflicted sector, if
+		 * whatever data read happens to match the checksum of 0, which is
+		 * rare, but has been observed on some disks.
+		 */
+		if (now > (getFluxmapDuration() - 12.5e6)) {
+			seekToIndexMark();
+			return 0;
+		}
+
+		nanoseconds_t clock = seekToPattern(SECTOR_SYNC_PATTERN);
+
+		auto syncDelta = tell().ns() - now;
+		/* Due to the weak nature of the Micropolis SYNC patern,
+		 * it's possible to detect a false SYNC during the gap
+		 * between the sector pulse and the write gate.  If the SYNC
+		 * is detected less than 100uS after the sector pulse, search
+		 * for another valid SYNC.
+		 *
+		 * Reference: Vector Micropolis Disk Controller Board Technical
+		 * Information Manual, pp. 1-16.
+		 */
+		if ((syncDelta > 0) && (syncDelta < 100e3)) {
+			seekToPattern(SECTOR_ADVANCE_PATTERN);
+			clock = seekToPattern(SECTOR_SYNC_PATTERN);
+		}
+
+		return clock;
 	}
 
 	void decodeSectorRecord()
 	{
-		readRawBits(16);
+		readRawBits(48);
 		auto rawbits = readRawBits(MICROPOLIS_ENCODED_SECTOR_SIZE*16);
 		auto bytes = decodeFmMfm(rawbits).slice(0, MICROPOLIS_ENCODED_SECTOR_SIZE);
 		ByteReader br(bytes);
 
-		br.read_8();  /* sync */
+		int syncByte = br.read_8();  /* sync */
+		if (syncByte != 0xFF)
+			return;
+
 		_sector->logicalTrack = br.read_8();
 		_sector->logicalSide = _sector->physicalHead;
 		_sector->logicalSector = br.read_8();
 		if (_sector->logicalSector > 15)
 			return;
-		if (_sector->logicalTrack > 77)
+		if (_sector->logicalTrack > 76)
+			return;
+		if (_sector->logicalTrack != _sector->physicalCylinder)
 			return;
 
 		br.read(10);  /* OS data or padding */
-		auto data = br.read(256);
+		auto data = br.read(MICROPOLIS_PAYLOAD_SIZE);
 		uint8_t wantChecksum = br.read_8();
-		uint8_t gotChecksum = micropolisChecksum(bytes.slice(1, 2+266));
+
+		/* If not specified, automatically determine the checksum type.
+		* Once the checksum type is determined, it will be used for the
+		* entire disk.
+		*/
+		if (_checksumType == 0) {
+			/* Calculate both standard Micropolis (MDOS, CP/M, OASIS) and MZOS checksums */
+			if (wantChecksum == micropolisChecksum(bytes.slice(1, 2+266))) {
+				_checksumType = 1;
+			} else if (wantChecksum == mzosChecksum(bytes.slice(MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE))) {
+				_checksumType = 2;
+				std::cout << "Note: MZOS checksum detected." << std::endl;
+			}
+		}
+
+		uint8_t gotChecksum;
+
+		if (_checksumType == 2) {
+			gotChecksum = mzosChecksum(bytes.slice(MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE));
+		} else {
+			gotChecksum = micropolisChecksum(bytes.slice(1, 2+266));
+		}
+
 		br.read(5);  /* 4 byte ECC and ECC-present flag */
 
-		if (_config.sector_output_size() == 256)
+		if (_config.sector_output_size() == MICROPOLIS_PAYLOAD_SIZE)
 			_sector->data = data;
 		else if (_config.sector_output_size() == MICROPOLIS_ENCODED_SECTOR_SIZE)
 			_sector->data = bytes;
@@ -69,8 +163,16 @@ public:
 			Error() << "Sector output size may only be 256 or 275";
 		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 	}
+
+	std::set<unsigned> requiredSectors(unsigned cylinder, unsigned head) const override
+	{
+		static std::set<unsigned> sectors = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+		return sectors;
+	}
+
 private:
 	const MicropolisDecoderProto& _config;
+	int _checksumType;	/* -1 = auto, 1 = Micropolis, 2=MZOS */
 };
 
 std::unique_ptr<AbstractDecoder> createMicropolisDecoder(const DecoderProto& config)

arch/micropolis/micropolis.h

@@ -1,7 +1,9 @@
 #ifndef MICROPOLIS_H
 #define MICROPOLIS_H
 
-#define MICROPOLIS_ENCODED_SECTOR_SIZE (1+2+266+6)
+#define MICROPOLIS_PAYLOAD_SIZE         (256)
+#define MICROPOLIS_HEADER_SIZE          (1+2+10)
+#define MICROPOLIS_ENCODED_SECTOR_SIZE  (MICROPOLIS_HEADER_SIZE + MICROPOLIS_PAYLOAD_SIZE + 6)
 
 class AbstractDecoder;
 class AbstractEncoder;

arch/micropolis/micropolis.proto

@@ -5,6 +5,8 @@ import "lib/common.proto";
 message MicropolisDecoderProto {
 	optional int32 sector_output_size = 1 [default = 256,
 		(help) = "How much of the raw sector should be saved. Must be 256 or 275"];
+	optional int32 checksum_type = 2 [default = 0,
+		(help) = "Checksum type to use: 0 = automatically detect, 1 = Micropolis, 2 = MZOS"];
 }
 
 message MicropolisEncoderProto {}

doc/disk-micropolis.md

@@ -54,6 +54,18 @@ fluxengine read micropolis630 vgi # double-sided Mod II
 You should end up with a `micropolis.vgi` instead. The format is well-defined
 for double-sided disks so post-processing is not necessary.
 
+While most operating systems use the standard Micropolis checksum, Vector
+Graphic MZOS uses a unique checksum.  The decoder will automatically detect
+the checksum type in use; however, a specific checksum type may be forced
+using the `--decoder.micropolis.checksum_type=n` where the type is one of:
+
+| Type | Description                             |
+|------|-----------------------------------------|
+| 0    | Automatically detect                    |
+| 1    | Standard Micropolis (MDOS, CP/M, OASIS) |
+| 2    | Vector Graphic MZOS                     |
+
+
 Writing disks
 -------------