Micropolis: Add Vector ECC support

arch/micropolis/decoder.cc

@@ -59,6 +59,70 @@ uint8_t mzosChecksum(const Bytes& bytes)
     return checksum;
 }
 
+static uint8_t b(uint32_t field, uint8_t pos)
+{
+    return (field >> pos) & 1;
+}
+
+static uint8_t eccNextBit(uint32_t ecc, uint8_t data_bit)
+{
+    // This is 0x81932080 which is 0x0104C981 with reversed bits
+    return b(ecc, 7) ^ b(ecc, 13) ^ b(ecc, 16) ^ b(ecc, 17) ^ b(ecc, 20)
+            ^ b(ecc, 23) ^ b(ecc, 24) ^ b(ecc, 31) ^ data_bit;
+}
+
+uint32_t vectorGraphicEcc(const Bytes& bytes)
+{
+    uint32_t e = 0;
+    Bytes payloadBytes = bytes.slice(0, bytes.size()-4);
+    ByteReader payload(payloadBytes);
+    while (!payload.eof()) {
+        uint8_t byte = payload.read_8();
+        for (int i = 0; i < 8; i++) {
+            e = (e << 1) | eccNextBit(e, byte >> 7);
+            byte <<= 1;
+        }
+    }
+    Bytes trailerBytes = bytes.slice(bytes.size()-4);
+    ByteReader trailer(trailerBytes);
+    uint32_t res = e;
+    while (!trailer.eof()) {
+        uint8_t byte = trailer.read_8();
+        for (int i = 0; i < 8; i++) {
+            res = (res << 1) | eccNextBit(e, byte >> 7);
+            e <<= 1;
+            byte <<= 1;
+        }
+    }
+    return res;
+}
+
+/* Fixes bytes when possible, returning true if changed. */
+static bool vectorGraphicEccFix(Bytes& bytes, uint32_t syndrome)
+{
+    uint32_t ecc = syndrome;
+    int pos = (MICROPOLIS_ENCODED_SECTOR_SIZE-5)*8+7;
+    bool aligned = false;
+    while ((ecc & 0xff000000) == 0) {
+        pos += 8;
+        ecc <<= 8;
+    }
+    for (; pos >= 0; pos--) {
+        bool bit = ecc & 1;
+        ecc >>= 1;
+        if (bit)
+            ecc ^= 0x808264c0;
+        if ((ecc & 0xff07ffff) == 0)
+            aligned = true;
+        if (aligned && pos % 8 == 0)
+            break;
+    }
+    if (pos < 0)
+        return false;
+    bytes[pos/8] ^= ecc >> 16;
+    return true;
+}
+
 class MicropolisDecoder : public Decoder
 {
 public:
@@ -132,6 +196,17 @@ public:
         auto rawbits = readRawBits(MICROPOLIS_ENCODED_SECTOR_SIZE * 16);
         auto bytes =
             decodeFmMfm(rawbits).slice(0, MICROPOLIS_ENCODED_SECTOR_SIZE);
+
+        bool eccPresent = bytes[274] == 0xaa;
+        uint32_t ecc = 0;
+        if (_config.ecc_type() == MicropolisDecoderProto::VECTOR && eccPresent) {
+            ecc = vectorGraphicEcc(bytes.slice(0, 274));
+            if (ecc != 0) {
+                vectorGraphicEccFix(bytes, ecc);
+                ecc = vectorGraphicEcc(bytes.slice(0, 274));
+            }
+        }
+
         ByteReader br(bytes);
 
         int syncByte = br.read_8(); /* sync */
@@ -193,8 +268,10 @@ public:
             _sector->data = bytes;
         else
             error("Sector output size may only be 256 or 275");
-        _sector->status =
+        if (wantChecksum == gotChecksum && (!eccPresent || ecc == 0))
-            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+            _sector->status = Sector::OK;
+        else
+            _sector->status = Sector::BAD_CHECKSUM;
     }
 
 private:

arch/micropolis/encoder.cc

@@ -8,7 +8,8 @@
 
 static void write_sector(std::vector<bool>& bits,
     unsigned& cursor,
-    const std::shared_ptr<const Sector>& sector)
+    const std::shared_ptr<const Sector>& sector,
+    MicropolisEncoderProto::EccType eccType)
 {
     if ((sector->data.size() != 256) &&
         (sector->data.size() != MICROPOLIS_ENCODED_SECTOR_SIZE))
@@ -45,8 +46,15 @@ static void write_sector(std::vector<bool>& bits,
             writer.write_8(0); /* Padding */
         writer += sector->data;
         writer.write_8(micropolisChecksum(sectorData.slice(1)));
-        for (int i = 0; i < 5; i++)
+
-            writer.write_8(0); /* 4 byte ECC and ECC not present flag */
+        uint8_t eccPresent = 0;
+        uint32_t ecc = 0;
+        if (eccType == MicropolisEncoderProto::VECTOR) {
+            eccPresent = 0xaa;
+            ecc = vectorGraphicEcc(sectorData + Bytes(4));
+        }
+        writer.write_be32(ecc);
+        writer.write_8(eccPresent);
     }
     for (uint8_t b : sectorData)
         fullSector->push_back(b);
@@ -93,7 +101,7 @@ public:
         for (const auto& sectorData : sectors) {
             indexes.push_back(cursor);
             prev_cursor = cursor;
-            write_sector(bits, cursor, sectorData);
+            write_sector(bits, cursor, sectorData, _config.ecc_type());
         }
         indexes.push_back(prev_cursor + (cursor - prev_cursor)/2);
         indexes.push_back(cursor);

arch/micropolis/micropolis.h

@@ -17,5 +17,6 @@ extern std::unique_ptr<Encoder> createMicropolisEncoder(
     const EncoderProto& config);
 
 extern uint8_t micropolisChecksum(const Bytes& bytes);
+extern uint32_t vectorGraphicEcc(const Bytes& bytes);
 
 #endif

arch/micropolis/micropolis.proto

@@ -8,17 +8,30 @@ message MicropolisDecoderProto {
 		MICROPOLIS = 1;
 		MZOS = 2;
 	}
+	enum EccType {
+		NONE = 0;
+		VECTOR = 1;
+	}
 
 	optional int32 sector_output_size = 1 [default = 256,
 		(help) = "How much of the raw sector should be saved. Must be 256 or 275"];
 	optional ChecksumType checksum_type = 2 [default = AUTO,
 		(help) = "Checksum type to use: AUTO, MICROPOLIS, MZOS"];
+	optional EccType ecc_type = 3 [default = NONE,
+		(help) = "ECC type to use: NONE, VECTOR"];
 }
 
 message MicropolisEncoderProto {
+    enum EccType {
+        NONE = 0;
+        VECTOR = 1;
+    }
+
     optional double clock_period_us = 1
         [ default = 2.0, (help) = "clock rate on the real device" ];
     optional double rotational_period_ms = 2
         [ default = 200.0, (help) = "rotational period on the real device" ];
+    optional EccType ecc_type = 3 [default = NONE,
+        (help) = "ECC type to use for IMG data: NONE, VECTOR"];
 }
 

doc/disk-micropolis.md

@@ -22,13 +22,25 @@ pinout as a 96tpi PC 5.25" drive. In use they should be identical.
 While most operating systems use the standard Micropolis checksum, Vector
 Graphic MZOS uses a unique checksum.  The decoder will automatically detect
 the checksum type in use; however, a specific checksum type may be forced
-using the `--decoder.micropolis.checksum_type=n` where the type is one of:
+using the `--decoder.micropolis.checksum_type=TYPE` where TYPE is one of:
 
-| Type | Description                             |
+| Checksum   | Description                             |
-|------|-----------------------------------------|
+|------------|-----------------------------------------|
-| 0    | Automatically detect                    |
+| AUTO       | Automatically detect                    |
-| 1    | Standard Micropolis (MDOS, CP/M, OASIS) |
+| MICROPOLIS | Standard Micropolis (MDOS, CP/M, OASIS) |
-| 2    | Vector Graphic MZOS                     |
+| MZOS       | Vector Graphic MZOS                     |
+
+Later versions of the Micropolis format supported ECC, especially in
+controllers with HDD support. The ECC can detect and correct errors. However,
+it is unclear what ECC algorithm was used by each vendor. ECC is disabled by
+default, but available for checking and correcting using
+`--decoder.micropolis.ecc_type=TYPE` and for writing from IMG files using
+`--encoder.micropolis.ecc_type=TYPE`, where TYPE is one of:
+
+| ECC    | Description                              |
+|--------|------------------------------------------|
+| NONE   | No ECC processing enabled                |
+| VECTOR | Vector Graphic Dual-Mode Disk Controller |
 
 The [CP/M BIOS](https://www.seasip.info/Cpm/bios.html) defined SELDSK, SETTRK,
 and SETSEC, but no function to select the head/side. Double-sided floppies

src/formats/micropolis.textpb

@@ -24,13 +24,25 @@ pinout as a 96tpi PC 5.25" drive. In use they should be identical.
 While most operating systems use the standard Micropolis checksum, Vector
 Graphic MZOS uses a unique checksum.  The decoder will automatically detect
 the checksum type in use; however, a specific checksum type may be forced
-using the `--decoder.micropolis.checksum_type=n` where the type is one of:
+using the `--decoder.micropolis.checksum_type=TYPE` where TYPE is one of:
 
-| Type | Description                             |
+| Checksum   | Description                             |
-|------|-----------------------------------------|
+|------------|-----------------------------------------|
-| 0    | Automatically detect                    |
+| AUTO       | Automatically detect                    |
-| 1    | Standard Micropolis (MDOS, CP/M, OASIS) |
+| MICROPOLIS | Standard Micropolis (MDOS, CP/M, OASIS) |
-| 2    | Vector Graphic MZOS                     |
+| MZOS       | Vector Graphic MZOS                     |
+
+Later versions of the Micropolis format supported ECC, especially in
+controllers with HDD support. The ECC can detect and correct errors. However,
+it is unclear what ECC algorithm was used by each vendor. ECC is disabled by
+default, but available for checking and correcting using
+`--decoder.micropolis.ecc_type=TYPE` and for writing from IMG files using
+`--encoder.micropolis.ecc_type=TYPE`, where TYPE is one of:
+
+| ECC    | Description                              |
+|--------|------------------------------------------|
+| NONE   | No ECC processing enabled                |
+| VECTOR | Vector Graphic Dual-Mode Disk Controller |
 
 The [CP/M BIOS](https://www.seasip.info/Cpm/bios.html) defined SELDSK, SETTRK,
 and SETSEC, but no function to select the head/side. Double-sided floppies