The new decoder architecture now works, at least for the FB100. All I need now

is to rewrite every single other decoder.

lib/decoders/decoders.cc

@@ -1,6 +1,7 @@
 #include "globals.h"
 #include "flags.h"
 #include "fluxmap.h"
+#include "fluxmapreader.h"
 #include "decoders.h"
 #include "record.h"
 #include "protocol.h"
@@ -49,14 +50,11 @@ static const std::string BLOCK_ELEMENTS[] =
 nanoseconds_t Fluxmap::guessClock() const
 {
     uint32_t buckets[256] = {};
-    size_t cursor = 0;
     FluxmapReader fr(*this);
-    while (cursor < bytes())
+
+    while (!fr.eof())
     {
-        unsigned interval;
+        unsigned interval = fr.readNextMatchingOpcode(F_OP_PULSE);
-        int opcode = fr.readPulse(interval);
-        if (opcode != 0x80)
-            break;
         if (interval > 0xff)
             continue;
         buckets[interval]++;
@@ -178,7 +176,7 @@ const RawBits Fluxmap::decodeToBits(nanoseconds_t clockPeriod) const
         for (;;)
         {
             unsigned interval;
-            int opcode = fr.read(interval);
+            int opcode = fr.readOpcode(interval);
             timestamp += interval * NS_PER_TICK;
             if (opcode == -1)
                 goto abort;
@@ -213,6 +211,22 @@ nanoseconds_t AbstractDecoder::guessClockImpl(Track& track) const
     return track.fluxmap->guessClock();
 }
 
+void AbstractSeparatedDecoder::decodeToSectors(Track& track)
+{
+    nanoseconds_t clockPeriod = guessClock(track);
+    if (clockPeriod == 0)
+    {
+        std::cout << "       no clock detected; giving up" << std::endl;
+        return;
+    }
+    std::cout << fmt::format("       {:.2f} us clock; ", (double)clockPeriod/1000.0) << std::flush;
+
+    const auto& bitmap = track.fluxmap->decodeToBits(clockPeriod);
+    std::cout << fmt::format("{} bytes encoded; ", bitmap.size()/8) << std::flush;
+
+    decodeToSectors(bitmap, track);
+}
+
 void AbstractSeparatedDecoder::decodeToSectors(const RawBits& bitmap, Track& track)
 {
     track.rawrecords = std::make_unique<RawRecordVector>(extractRecords(bitmap));

lib/decoders/decoders.h

@@ -28,7 +28,7 @@ public:
     nanoseconds_t guessClock(Track& track) const;
     virtual nanoseconds_t guessClockImpl(Track& track) const;
 
-    virtual void decodeToSectors(const RawBits& bitmap, Track& track) = 0;
+    virtual void decodeToSectors(Track& track) = 0;
 };
 
 /* DEPRECATED */
@@ -41,6 +41,7 @@ public:
     virtual SectorVector decodeToSectors(const RawRecordVector& rawrecords,
             unsigned physicalTrack, unsigned physicalSide) = 0;
 
+    void decodeToSectors(Track& track);
     void decodeToSectors(const RawBits& bitmap, Track& track);
 };
 

lib/decoders/fluxmapreader.cc

@@ -0,0 +1,150 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "fluxmapreader.h"
+#include "flags.h"
+#include "protocol.h"
+#include <numeric>
+#include <math.h>
+
+static DoubleFlag clockDecodeThreshold(
+    { "--bit-error-threshold" },
+    "Amount of error to tolerate in pulse timing.",
+    0.20);
+
+int FluxmapReader::readOpcode(unsigned& ticks)
+{
+    ticks = 0;
+
+    while (!eof())
+    {
+        uint8_t b = _bytes[_pos.bytes++];
+        if (b < 0x80)
+            ticks += b;
+        else
+        {
+            _pos.ticks += ticks;
+            return b;
+        }
+    }
+
+    _pos.ticks += ticks;
+    return -1;
+}
+
+unsigned FluxmapReader::readNextMatchingOpcode(uint8_t opcode)
+{
+    unsigned ticks = 0;
+
+    for (;;)
+    {
+        unsigned thisTicks;
+        int op = readOpcode(thisTicks);
+        ticks += thisTicks;
+        if (op == -1)
+            return 0;
+        if (op == opcode)
+            return ticks;
+    }
+}
+
+FluxPattern::FluxPattern(unsigned len, uint64_t pattern)
+{
+    const uint64_t TOPBIT = 1ULL << 63;
+
+    assert(pattern != 0);
+    bits = len;
+
+    while (!(pattern & TOPBIT))
+        pattern <<= 1;
+
+    length = 0;
+    while (pattern != TOPBIT)
+    {
+        unsigned interval = 0;
+        do
+        {
+            pattern <<= 1;
+            interval++;
+        }
+        while (!(pattern & TOPBIT));
+        intervals.push_back(interval);
+        length += interval;
+    }
+}
+
+bool FluxPattern::matches(const unsigned* end, double& clock) const
+{
+    const unsigned* start = end - intervals.size();
+    unsigned candidatelength = std::accumulate(start, end, 0);
+    clock = (double)candidatelength / (double)length;
+
+    for (unsigned i=0; i<intervals.size(); i++)
+    {
+        double ii = clock * (double)intervals[i];
+        double ci = (double)start[i];
+        double error = fabs(1.0 - ii/ci);
+        if (error > clockDecodeThreshold)
+            return false;
+    }
+
+    return true;
+}
+
+nanoseconds_t FluxmapReader::seekToPattern(const FluxPattern& pattern)
+{
+    unsigned length = pattern.intervals.size();
+    unsigned candidates[length];
+    Position positions[length];
+
+    for (unsigned& i : candidates)
+        i = 0;
+    for (Position& p : positions)
+        p = tell();
+
+    while (!eof())
+    {
+        unsigned interval = readNextMatchingOpcode(F_OP_PULSE);
+
+        Position current = positions[0];
+        for (unsigned i=0; i<(length-1); i++)
+        {
+            positions[i] = positions[i+1];
+            candidates[i] = candidates[i+1];
+        }
+        positions[length-1] = tell();
+        candidates[length-1] = interval;
+
+        double clock;
+        if (pattern.matches(&candidates[length], clock))
+        {
+            seek(current);
+            return clock * NS_PER_TICK;
+        }
+    }
+
+    return 0;
+}
+
+bool FluxmapReader::readRawBit(nanoseconds_t clockPeriod)
+{
+    if (_pendingZeroBits)
+    {
+        _pendingZeroBits--;
+        return false;
+    }
+
+    unsigned interval = readNextMatchingOpcode(F_OP_PULSE);
+    unsigned clockTicks = clockPeriod / NS_PER_TICK;
+    double clocks = (double)interval / clockTicks;
+
+    _pendingZeroBits = (int)round(clocks) - 1;
+    return true;
+}
+
+std::vector<bool> FluxmapReader::readRawBits(unsigned count, nanoseconds_t clockPeriod)
+{
+    std::vector<bool> result;
+    while (!eof() && count--)
+        result.push_back(readRawBit(clockPeriod));
+    return result;
+}

lib/decoders/fluxmapreader.h

@@ -0,0 +1,76 @@
+#ifndef FLUXMAPREADER_H
+#define FLUXMAPREADER_H
+
+#include "fluxmap.h"
+#include "protocol.h"
+
+class FluxPattern
+{
+public:
+    FluxPattern(unsigned len, uint64_t pattern);
+
+    bool matches(const unsigned* intervals, double& clock) const;
+
+    unsigned bits;
+    std::vector<unsigned> intervals;
+    unsigned length;
+};
+
+class FluxmapReader
+{
+public:
+    struct Position
+    {
+        unsigned bytes;
+        unsigned ticks;
+    };
+
+public:
+    FluxmapReader(const Fluxmap& fluxmap):
+        _fluxmap(fluxmap),
+        _bytes(fluxmap.ptr()),
+        _size(fluxmap.bytes())
+    {
+        rewind();
+    }
+
+    FluxmapReader(const Fluxmap&& fluxmap) = delete;
+
+    void rewind()
+    {
+        _pos.bytes = 0;
+        _pos.ticks = 0;
+        _pendingZeroBits = 0;
+    }
+
+    bool eof() const
+    { return _pos.bytes == _size; }
+
+    Position tell() const
+    { return _pos; }
+
+    void seek(const Position& pos)
+    { _pos = pos; }
+
+    nanoseconds_t tellNs() const
+    { return _pos.ticks * NS_PER_TICK; }
+
+    int readOpcode(unsigned& ticks);
+    unsigned readNextMatchingOpcode(uint8_t opcode);
+
+    void seek(nanoseconds_t ns);
+    void seekToIndexMark();
+    nanoseconds_t seekToPattern(const FluxPattern& pattern);
+
+    bool readRawBit(nanoseconds_t clockPeriod);
+    std::vector<bool> readRawBits(unsigned count, nanoseconds_t clockPeriod);
+
+private:
+    const Fluxmap& _fluxmap;
+    const uint8_t* _bytes;
+    const size_t _size;
+    Position _pos;
+    unsigned _pendingZeroBits;
+};
+
+#endif

lib/fb100/decoder.cc

@@ -1,5 +1,6 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
 #include "decoders.h"
@@ -13,23 +14,7 @@
 #include <string.h>
 #include <algorithm>
 
-static bool search(const RawBits& rawbits, size_t& cursor)
+const FluxPattern SECTOR_ID_PATTERN(16, 0xabaa);
-{
-    uint16_t fifo = 0;
-
-    while (cursor < rawbits.size())
-    {
-        fifo = (fifo << 1) | rawbits[cursor++];
-        
-        if (fifo == 0xabaa)
-        {
-            cursor -= 16;
-            return true;
-        }
-    }
-
-    return false;
-}
 
 /* 
  * Reverse engineered from a dump of the floppy drive's ROM. I have no idea how
@@ -114,34 +99,33 @@ static uint16_t checksum(const Bytes& bytes)
     return (crchi << 8) | crclo;
 }
 
-void Fb100Decoder::decodeToSectors(const RawBits& rawbits, Track& track)
+void Fb100Decoder::decodeToSectors(Track& track)
 {
-    size_t cursor = 0;
-
     if (track.rawrecords || track.sectors)
         Error() << "cannot decode track twice";
     track.rawrecords = std::make_unique<RawRecordVector>();
     track.sectors = std::make_unique<SectorVector>();
 
+    FluxmapReader fmr(*track.fluxmap);
+
     for (;;)
     {
-        if (!search(rawbits, cursor))
+        nanoseconds_t clockPeriod = fmr.seekToPattern(SECTOR_ID_PATTERN);
+        if (fmr.eof())
             break;
 
-        unsigned record_start = cursor;
+        auto rawbits = fmr.readRawBits(FB100_RECORD_SIZE*16, clockPeriod);
-        cursor = std::min(cursor + FB100_RECORD_SIZE*16, rawbits.size());
-        std::vector<bool> recordbits(rawbits.begin() + record_start, rawbits.begin() + cursor);
 
         track.rawrecords->push_back(
             std::unique_ptr<RawRecord>(
                 new RawRecord(
-                    record_start,
+                    0,
-                    recordbits.begin(),
+                    rawbits.begin(),
-                    recordbits.end())
+                    rawbits.end())
             )
         );
 
-        const Bytes bytes = decodeFmMfm(recordbits).slice(0, FB100_RECORD_SIZE);
+        const Bytes bytes = decodeFmMfm(rawbits).slice(0, FB100_RECORD_SIZE);
         ByteReader br(bytes);
         br.seek(1);
         const Bytes id = br.read(FB100_ID_SIZE);
@@ -164,6 +148,7 @@ void Fb100Decoder::decodeToSectors(const RawBits& rawbits, Track& track)
         int status = (wantPayloadCrc == gotPayloadCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
         auto sector = std::unique_ptr<Sector>(
             new Sector(status, trackid, 0, sectorid, data));
+        sector->clock = clockPeriod;
         track.sectors->push_back(std::move(sector));
     }
 }

lib/fb100/fb100.h

@@ -14,7 +14,7 @@ class Fb100Decoder : public AbstractStatefulDecoder
 public:
     virtual ~Fb100Decoder() {}
 
-    void decodeToSectors(const RawBits& bitmap, Track& track);
+    void decodeToSectors(Track& track) override;
 };
 
 #endif

lib/fluxmap.cc

@@ -77,33 +77,3 @@ void Fluxmap::precompensate(int threshold_ticks, int amount_ticks)
         }
     }
 }
-
-int FluxmapReader::read(unsigned& ticks)
-{
-    ticks = 0;
-
-    while (_cursor < _size)
-    {
-        uint8_t b = _bytes[_cursor++];
-        if (b < 0x80)
-            ticks += b;
-        else
-            return b;
-    }
-
-    return -1;
-}
-
-int FluxmapReader::readPulse(unsigned& ticks)
-{
-    ticks = 0;
-
-    for (;;)
-    {
-        unsigned thisTicks;
-        int opcode = read(thisTicks);
-        ticks += thisTicks;
-        if ((opcode == -1) || (opcode == 0x80))
-            return opcode;
-    }
-}

lib/fluxmap.h

@@ -44,33 +44,4 @@ private:
     Bytes _bytes;
 };
 
-class FluxmapReader
-{
-public:
-    FluxmapReader(const Fluxmap& fluxmap):
-        _fluxmap(fluxmap),
-        _bytes(fluxmap.ptr()),
-        _size(fluxmap.bytes())
-    {
-        rewind();
-    }
-
-    FluxmapReader(const Fluxmap&& fluxmap) = delete;
-
-    void rewind()
-    { _cursor = 0; }
-
-    size_t tell() const
-    { return _cursor; }
-
-    int read(unsigned& ticks);
-    int readPulse(unsigned& ticks);
-
-private:
-    const Fluxmap& _fluxmap;
-    const uint8_t* _bytes;
-    const size_t _size;
-    size_t _cursor;
-};
-
 #endif

lib/reader.cc

@@ -40,7 +40,7 @@ static IntFlag retries(
 	5);
 
 static SettableFlag highDensityFlag(
-	{ "--high-density", "-H" },
+	{ "--high-density", "--hd" },
 	"set the drive to high density mode");
 
 static sqlite3* outdb;
@@ -147,22 +147,20 @@ void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename
 		for (int retry = ::retries; retry >= 0; retry--)
 		{
 			track->readFluxmap();
+			decoder.decodeToSectors(*track);
 
-			nanoseconds_t clockPeriod = decoder.guessClock(*track);
+			if (!track->sectors)
-			if (clockPeriod == 0)
 			{
-				std::cout << "       no clock detected; giving up" << std::endl;
+				std::cout << "       no sectors found; giving up" << std::endl;
 				continue;
 			}
-			std::cout << fmt::format("       {:.2f} us clock; ", (double)clockPeriod/1000.0) << std::flush;
 
-			const auto& bitmap = track->fluxmap->decodeToBits(clockPeriod);
+			std::cout << fmt::format("       {} records, {} sectors; ",
-			std::cout << fmt::format("{} bytes encoded; ", bitmap.size()/8) << std::flush;
+				 track->rawrecords->size(),
-
+			     track->sectors->size());
-			decoder.decodeToSectors(bitmap, *track);
+			if (track->sectors->size() > 0)
-
+				std::cout << fmt::format("{:.2}us clock; ",
-			std::cout << fmt::format("{} records", track->rawrecords->size()) << std::endl
+					track->sectors->begin()->get()->clock / 1000.0);
-			          << "       " << track->sectors->size() << " sectors; ";
 
 			for (auto& sector : *track->sectors)
 			{
@@ -191,7 +189,7 @@ void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename
 				std::cout << "\nRaw (undecoded) records follow:\n\n";
 				for (auto& record : *track->rawrecords)
 				{
-					std::cout << fmt::format("I+{:.3f}ms", (double)(record->position*clockPeriod)/1e6)
+					std::cout << fmt::format("I+0x{:x} bits", record->position)
 					          << std::endl;
 					hexdump(std::cout, toBytes(record->data));
 					std::cout << std::endl;

lib/sector.h

@@ -30,6 +30,7 @@ public:
     {}
 
 	int status;
+    nanoseconds_t clock;
     const int track;
     const int side;
     const int sector;

meson.build

@@ -92,7 +92,7 @@ decoderlib = declare_dependency(
             [
                 'lib/decoders/decoders.cc',
                 'lib/decoders/fmmfm.cc',
-                'lib/decoders/fluxreader.cc',
+                'lib/decoders/fluxmapreader.cc',
             ],
             dependencies: [fmtlib, felib]
         ),
@@ -267,3 +267,4 @@ test('Kryoflux',        executable('kryoflux-test', ['tests/kryoflux.cc'],
 test('Compression',     executable('compression-test', ['tests/compression.cc'],       dependencies: [felib, decoderlib]))
 test('Bytes',           executable('bytes-test', ['tests/bytes.cc'],                   dependencies: [felib]))
 test('Crunch',          executable('crunch-test', ['tests/crunch.cc'],                 dependencies: [felib, crunchlib]))
+test('FluxPattern',     executable('fluxpattern-test', ['tests/fluxpattern.cc'],       dependencies: [felib, decoderlib]))

protocol.h

@@ -73,6 +73,12 @@ enum
     F_ERROR_INTERNAL,
 };
 
+enum
+{
+    F_OP_PULSE = 0x80,
+    F_OP_INDEX = 0x81
+};
+
 struct frame_header
 {
     uint8_t type;

src/fe-inspect.cc

@@ -2,6 +2,7 @@
 #include "flags.h"
 #include "reader.h"
 #include "fluxmap.h"
+#include "fluxmapreader.h"
 #include "decoders.h"
 #include "image.h"
 #include "protocol.h"
@@ -73,13 +74,9 @@ int main(int argc, const char* argv[])
 		nanoseconds_t seekto = seekFlag*1000000.0;
 		int ticks = 0;
 		FluxmapReader fr(*track->fluxmap);
-		for (;;)
+		while (!fr.eof())
 		{
-			unsigned interval;
+			ticks += fr.readNextMatchingOpcode(F_OP_PULSE);
-			int opcode = fr.readPulse(interval);
-			if (opcode == -1)
-				break;
-			ticks += interval;
 			
 			now = ticks * NS_PER_TICK;
 			if (now >= seekto)
@@ -89,13 +86,9 @@ int main(int argc, const char* argv[])
 		std::cout << fmt::format("{: 10.3f}:-", ticks*US_PER_TICK);
 		nanoseconds_t lasttransition = 0;
 		fr.rewind();
-		for (;;)
+		while (!fr.eof())
 		{
-			unsigned interval;
+			ticks += fr.readNextMatchingOpcode(F_OP_PULSE);
-			int opcode = fr.readPulse(interval);
-			if (opcode == -1)
-				break;
-			ticks += interval;
 
 			nanoseconds_t transition = ticks*NS_PER_TICK;
 			nanoseconds_t next;
@@ -116,7 +109,7 @@ int main(int argc, const char* argv[])
 
 			nanoseconds_t length = transition - lasttransition;
 			std::cout << fmt::format("==== {:06x} {: 10.3f} +{:.3f} = {:.1f} clocks",
-			    fr.tell(),
+			    fr.tell().bytes,
 				(double)transition / 1000.0,
 				(double)length / 1000.0,
 				(double)length / clockPeriod);

tests/fluxpattern.cc

@@ -0,0 +1,69 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "fluxmapreader.h"
+#include <assert.h>
+
+typedef std::vector<unsigned> ivector;
+
+static void test_patternconstruction()
+{
+    {
+        FluxPattern fp(16, 0x0003);
+        assert(fp.bits == 16);
+        assert(fp.intervals == ivector{ 1 });
+    }
+
+    {
+        FluxPattern fp(16, 0xc000);
+        assert(fp.bits == 16);
+        assert(fp.intervals == ivector{ 1 });
+    }
+
+    {
+        FluxPattern fp(16, 0x0050);
+        assert(fp.bits == 16);
+        assert(fp.intervals == ivector{ 2 });
+    }
+
+    {
+        FluxPattern fp(16, 0x0070);
+        assert(fp.bits == 16);
+        assert((fp.intervals == ivector{ 1, 1 }));
+    }
+
+    {
+        FluxPattern fp(16, 0x0070);
+        assert(fp.bits == 16);
+        assert((fp.intervals == ivector{ 1, 1 }));
+    }
+
+    {
+        FluxPattern fp(16, 0x0110);
+        assert(fp.bits == 16);
+        assert((fp.intervals == ivector{ 4 }));
+    }
+}
+
+static void test_patternmatching()
+{
+    FluxPattern fp(16, 0x000b);
+    const unsigned matching[] = { 100, 100, 200, 100 };
+    const unsigned notmatching[] = { 100, 200, 100, 100 };
+    const unsigned closematch1[] = { 90, 90, 180, 90 };
+    const unsigned closematch2[] = { 110, 110, 220, 110 };
+    
+    double clock;
+    assert(fp.matches(&matching[4], clock));
+    assert(!fp.matches(&notmatching[4], clock));
+    assert(fp.matches(&closematch1[4], clock));
+    assert(fp.matches(&closematch2[4], clock));
+}
+
+int main(int argc, const char* argv[])
+{
+    test_patternconstruction();
+    test_patternmatching();
+    return 0;
+}
+
+