Split the encoder's use of clock and bitcell, as for FM/MFM they're different

things; make the FM/MFM decoders use twice the bitcell for the clock.

.github/workflows/ccpp.yml

@@ -21,7 +21,7 @@ jobs:
       with:
         fetch-depth: 1
     - name: brew
-      run: brew install sqlite pkg-config libusb ninja protobuf
+      run: brew install sqlite pkg-config libusb ninja protobuf truncate
     - name: make
       run: make
 

Makefile

@@ -1,6 +1,8 @@
 PACKAGES = zlib sqlite3 libusb-1.0 protobuf
 
-export CFLAGS = -x c++ --std=c++14 -ffunction-sections -fdata-sections
+export CFLAGS = -x c++ --std=gnu++2a -ffunction-sections -fdata-sections \
+	-Wno-deprecated-enum-enum-conversion \
+	-Wno-deprecated-enum-float-conversion
 export LDFLAGS = -pthread
 
 export COPTFLAGS = -Os
@@ -50,7 +52,7 @@ CFLAGS += -Ilib -Idep/fmt -Iarch
 export OBJDIR = .obj
 
 all: .obj/build.ninja
-	@ninja -f .obj/build.ninja
+	@ninja -f .obj/build.ninja -k 0
 	@if command -v cscope > /dev/null; then cscope -bRq; fi
 
 clean:

README.md

@@ -102,7 +102,6 @@ people who've had it work).
 | [Brother 120kB](doc/disk-brother.md)      |  🦄   |   🦖   |                                     |
 | [Brother 240kB](doc/disk-brother.md)      |  🦄   |   🦄   |                                     |
 | [Brother FB-100](doc/disk-fb100.md)       |  🦖   |        | Tandy Model 100, Husky Hunter, knitting machines |
-| [Kaypro II](doc/disk-ibm.md)              |  🦄   |   🦄   |                                     |
 | [Macintosh 800kB](doc/disk-macintosh.md)  |  🦄   |   🦄   | and probably the 400kB too          |
 | [TRS-80](doc/disk-trs80.md)               |  🦖   |   🦖*  | a minor variation of the IBM scheme |
 {: .datatable }

arch/aeslanier/decoder.cc

@@ -32,8 +32,9 @@ public:
 
     RecordType advanceToNextRecord()
 	{
-		_sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
-		if (_fmr->eof() || !_sector->clock)
+		_sector->bitcell = _fmr->seekToPattern(SECTOR_PATTERN);
+		_sector->clock = _sector->bitcell * 2;
+		if (_fmr->eof() || !_sector->bitcell)
 			return UNKNOWN_RECORD;
 		return SECTOR_RECORD;
 	}

arch/amiga/decoder.cc

@@ -8,7 +8,6 @@
 #include "bytes.h"
 #include "fmt/format.h"
 #include "lib/decoders/decoders.pb.h"
-#include "lib/data.pb.h"
 #include <string.h>
 #include <algorithm>
 
@@ -30,15 +29,16 @@ public:
 		_config(config.amiga())
 	{}
 
-    RecordType advanceToNextRecord()
+    RecordType advanceToNextRecord() override
 	{
-		_sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
-		if (_fmr->eof() || !_sector->clock)
+		_sector->bitcell = _fmr->seekToPattern(SECTOR_PATTERN);
+		_sector->clock = _sector->bitcell * 2;
+		if (_fmr->eof() || !_sector->bitcell)
 			return UNKNOWN_RECORD;
 		return SECTOR_RECORD;
 	}
 
-    void decodeSectorRecord()
+    void decodeSectorRecord() override
 	{
 		const auto& rawbits = readRawBits(AMIGA_RECORD_SIZE*16);
 		if (rawbits.size() < (AMIGA_RECORD_SIZE*16))

arch/apple2/decoder.cc

@@ -74,7 +74,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/brother/decoder.cc

@@ -63,7 +63,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/brother/encoder.cc

@@ -144,7 +144,7 @@ public:
 	}
 
     std::unique_ptr<Fluxmap> encode(int physicalTrack, int physicalSide,
-			const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image)
+			const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image) override
 	{
 		int logicalTrack;
 		if (physicalSide != 0)

arch/c64/decoder.cc

@@ -61,7 +61,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/c64/encoder.cc

@@ -231,7 +231,7 @@ public:
 	}
 
     std::unique_ptr<Fluxmap> encode(int physicalTrack, int physicalSide,
-            const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image)
+            const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image) override
     {
         /* The format ID Character # 1 and # 2 are in the .d64 image only present
          * in track 18 sector zero which contains the BAM info in byte 162 and 163.

arch/f85/decoder.cc

@@ -61,7 +61,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/fb100/decoder.cc

@@ -107,7 +107,8 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(SECTOR_ID_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(SECTOR_ID_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 		if (matcher == &SECTOR_ID_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		return RecordType::UNKNOWN_RECORD;

arch/ibm/decoder.cc

@@ -98,16 +98,18 @@ public:
 		_config(config.ibm())
     {}
 
-    RecordType advanceToNextRecord()
+    RecordType advanceToNextRecord() override
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 
 		/* If this is the MFM prefix byte, the the decoder is going to expect three
 		 * extra bytes on the front of the header. */
 		_currentHeaderLength = (matcher == &MFM_PATTERN) ? 3 : 0;
 
 		Fluxmap::Position here = tell();
+		resetFluxDecoder();
 		if (_currentHeaderLength > 0)
 			readRawBits(_currentHeaderLength*16);
 		auto idbits = readRawBits(16);
@@ -129,7 +131,7 @@ public:
 		return RecordType::UNKNOWN_RECORD;
 	}
 
-    void decodeSectorRecord()
+    void decodeSectorRecord() override
 	{
 		unsigned recordSize = _currentHeaderLength + IBM_IDAM_LEN;
 		auto bits = readRawBits(recordSize*16);
@@ -151,9 +153,11 @@ public:
 			_sector->logicalSide ^= 1;
 		if (_config.ignore_side_byte())
 			_sector->logicalSide = _sector->physicalHead;
+		if (_config.ignore_track_byte())
+			_sector->logicalTrack = _sector->physicalCylinder;
 	}
 
-    void decodeDataRecord()
+    void decodeDataRecord() override
 	{
 		unsigned recordLength = _currentHeaderLength + _currentSectorSize + 3;
 		auto bits = readRawBits(recordLength*16);

arch/ibm/encoder.cc

@@ -133,18 +133,24 @@ public:
 				encodeMfm(_bits, _cursor, bytes, _lastBit);
 		};
 
+		auto writeFillerRawBytes = [&](int count, uint16_t byte)
+		{
+			for (int i=0; i<count; i++)
+				writeRawBits(byte, 16);
+		};
+
 		auto writeFillerBytes = [&](int count, uint8_t byte)
 		{
-			Bytes bytes = { byte };
+			Bytes b { byte };
 			for (int i=0; i<count; i++)
-				writeBytes(bytes);
+				writeBytes(b);
 		};
 
 		double clockRateUs = 1e3 / trackdata.clock_rate_khz();
 		if (!trackdata.use_fm())
 			clockRateUs /= 2.0;
 		int bitsPerRevolution = (trackdata.track_length_ms() * 1000.0) / clockRateUs;
-		_bits.resize(bitsPerRevolution*2);
+		_bits.resize(bitsPerRevolution);
 		_cursor = 0;
 
 		uint8_t idamUnencoded = decodeUint16(trackdata.idam_byte());
@@ -160,9 +166,9 @@ public:
 			}
 		}
 
-		uint8_t gapFill = trackdata.use_fm() ? 0x00 : 0x4e;
+		uint16_t gapFill = trackdata.gap_fill_byte();
 
-		writeFillerBytes(trackdata.gap0(), gapFill);
+		writeFillerRawBytes(trackdata.gap0(), gapFill);
 		if (trackdata.emit_iam())
 		{
 			writeFillerBytes(trackdata.use_fm() ? 6 : 12, 0x00);
@@ -172,7 +178,7 @@ public:
 					writeRawBits(MFM_IAM_SEPARATOR, 16);
 			}
 			writeRawBits(trackdata.use_fm() ? FM_IAM_RECORD : MFM_IAM_RECORD, 16);
-			writeFillerBytes(trackdata.gap1(), gapFill);
+			writeFillerRawBytes(trackdata.gap1(), gapFill);
 		}
 
 		int logicalSide = physicalSide ^ trackdata.swap_sides();
@@ -180,7 +186,7 @@ public:
 		for (int sectorId : trackdata.sectors().sector())
 		{
 			if (!first)
-				writeFillerBytes(trackdata.gap3(), gapFill);
+				writeFillerRawBytes(trackdata.gap3(), gapFill);
 			first = false;
 
 			const auto& sectorData = image.get(physicalTrack, logicalSide, sectorId);
@@ -227,7 +233,7 @@ public:
 				writeBytes(header.slice(conventionalHeaderStart));
 			}
 
-			writeFillerBytes(trackdata.gap2(), gapFill);
+			writeFillerRawBytes(trackdata.gap2(), gapFill);
 
 			{
 				Bytes data;
@@ -261,15 +267,12 @@ public:
 			}
 		}
 
-		if (_cursor >= bitsPerRevolution)
-			Error() << fmt::format("track data overrun by {} bits ({:.3} ms)",
-					_cursor - bitsPerRevolution,
-					(_cursor - bitsPerRevolution) / (clockRateUs*1000.0));
+		if (_cursor >= _bits.size())
+			Error() << "track data overrun";
 		while (_cursor < _bits.size())
-			writeFillerBytes(1, gapFill);
+			writeFillerRawBytes(1, gapFill);
 
 		std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
-		_bits.resize(bitsPerRevolution);
 		fluxmap->appendBits(_bits, clockRateUs*1e3);
 		return fluxmap;
 	}

arch/ibm/ibm.proto

@@ -2,19 +2,20 @@ syntax = "proto2";
 
 import "lib/common.proto";
 
-// Next: 6
+// Next: 7
 message IbmDecoderProto {
 	message SectorsProto {
 		repeated int32 sector = 1            [(help) = "require these sectors to exist for a good read"];
 	}
 
 	optional bool ignore_side_byte = 2       [default = false, (help) = "ignore side byte in sector header"];
+	optional bool ignore_track_byte = 6      [default = false, (help) = "ignore track byte in sector header"];
 	optional bool swap_sides = 4             [default = false, (help) = "put logical side 1 on physical side 0"];
 	optional SectorsProto sectors = 5        [(help) = "require these sectors to exist for a good read"];
 }
 
 message IbmEncoderProto {
-	// Next: 18
+	// Next: 19
 	message TrackdataProto {
 		message SectorsProto {
 			repeated int32 sector = 1		[(help) = "write these sectors (in order) on each track"];
@@ -36,6 +37,7 @@ message IbmEncoderProto {
 		optional int32 gap3 = 12            [default=80, (help) = "size of gap 4 (the post-data or format gap)"];
 		optional bool swap_sides = 14       [default=false, (help) = "swap side bytes when writing"];
 		optional SectorsProto sectors = 17	[(help) = "write these sectors (in order) on each track"];
+		optional int32 gap_fill_byte = 18   [default=0x9254, (help) = "16-bit raw bit pattern of gap fill byte"];
 	}
 
 	repeated TrackdataProto trackdata = 1;

arch/macintosh/decoder.cc

@@ -132,7 +132,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/micropolis/decoder.cc

@@ -11,7 +11,7 @@
 static const FluxPattern SECTOR_SYNC_PATTERN(32, 0xaaaa5555);
 
 /* Adds all bytes, with carry. */
-static uint8_t checksum(const Bytes& bytes) {
+uint8_t micropolisChecksum(const Bytes& bytes) {
 	ByteReader br(bytes);
 	uint16_t sum = 0;
 	while (!br.eof()) {
@@ -35,9 +35,9 @@ public:
 	{
 		_fmr->seekToIndexMark();
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(SECTOR_SYNC_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(SECTOR_SYNC_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 		if (matcher == &SECTOR_SYNC_PATTERN) {
-			readRawBits(16);
 			return SECTOR_RECORD;
 		}
 		return UNKNOWN_RECORD;
@@ -45,6 +45,7 @@ public:
 
 	void decodeSectorRecord()
 	{
+		readRawBits(16);
 		auto rawbits = readRawBits(MICROPOLIS_ENCODED_SECTOR_SIZE*16);
 		auto bytes = decodeFmMfm(rawbits).slice(0, MICROPOLIS_ENCODED_SECTOR_SIZE);
 		ByteReader br(bytes);
@@ -61,7 +62,7 @@ public:
 		br.read(10);  /* OS data or padding */
 		_sector->data = br.read(256);
 		uint8_t wantChecksum = br.read_8();
-		uint8_t gotChecksum = checksum(bytes.slice(1, 2+266));
+		uint8_t gotChecksum = micropolisChecksum(bytes.slice(1, 2+266));
 		br.read(5);  /* 4 byte ECC and ECC-present flag */
 
 		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;

arch/micropolis/encoder.cc

@@ -0,0 +1,113 @@
+#include "globals.h"
+#include "micropolis.h"
+#include "sector.h"
+#include "decoders/decoders.h"
+#include "encoders/encoders.h"
+#include "image.h"
+#include "lib/encoders/encoders.pb.h"
+
+static void write_sector(std::vector<bool>& bits, unsigned& cursor, const std::shared_ptr<Sector>& sector)
+{
+	if ((sector->data.size() != 256) && (sector->data.size() != MICROPOLIS_ENCODED_SECTOR_SIZE))
+		Error() << "unsupported sector size --- you must pick 256 or 275";
+
+	int fullSectorSize = 40 + MICROPOLIS_ENCODED_SECTOR_SIZE + 40 + 35;
+	auto fullSector = std::make_shared<std::vector<uint8_t>>();
+	fullSector->reserve(fullSectorSize);
+	/* sector preamble */
+	for (int i=0; i<40; i++)
+		fullSector->push_back(0);
+	Bytes sectorData;
+	if (sector->data.size() == MICROPOLIS_ENCODED_SECTOR_SIZE)
+		sectorData = sector->data;
+	else
+	{
+		ByteWriter writer(sectorData);
+		writer.write_8(0xff); /* Sync */
+		writer.write_8(sector->logicalTrack);
+		writer.write_8(sector->logicalSector);
+		for (int i=0; i<10; i++)
+			writer.write_8(0); /* Padding */
+		writer += sector->data;
+		writer.write_8(micropolisChecksum(sectorData.slice(1)));
+		for (int i=0; i<5; i++)
+			writer.write_8(0); /* 4 byte ECC and ECC not present flag */
+	}
+	for (uint8_t b : sectorData)
+		fullSector->push_back(b);
+	/* sector postamble */
+	for (int i=0; i<40; i++)
+		fullSector->push_back(0);
+	/* filler */
+	for (int i=0; i<35; i++)
+		fullSector->push_back(0);
+
+	if (fullSector->size() != fullSectorSize)
+		Error() << "sector mismatched length";
+	bool lastBit = false;
+	encodeMfm(bits, cursor, fullSector, lastBit);
+	/* filler */
+	for (int i=0; i<5; i++)
+	{
+		bits[cursor++] = 1;
+		bits[cursor++] = 0;
+	}
+}
+
+class MicropolisEncoder : public AbstractEncoder
+{
+public:
+	MicropolisEncoder(const EncoderProto& config):
+		AbstractEncoder(config),
+		_config(config.micropolis())
+	{}
+
+	std::vector<std::shared_ptr<Sector>> collectSectors(int physicalTrack, int physicalSide, const Image& image) override
+	{
+		std::vector<std::shared_ptr<Sector>> sectors;
+
+		if ((physicalTrack >= 0) && (physicalTrack < 77))
+		{
+			for (int sectorId = 0; sectorId < 16; sectorId++)
+			{
+				const auto& sector = image.get(physicalTrack, physicalSide, sectorId);
+				if (sector)
+					sectors.push_back(sector);
+			}
+		}
+
+		return sectors;
+	}
+
+	std::unique_ptr<Fluxmap> encode(int physicalTrack, int physicalSide,
+			const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image) override
+	{
+		int bitsPerRevolution = 100000;
+		double clockRateUs = 2.00;
+
+		if ((physicalTrack < 0) || (physicalTrack >= 77) || sectors.empty())
+			return std::unique_ptr<Fluxmap>();
+
+		std::vector<bool> bits(bitsPerRevolution);
+		unsigned cursor = 0;
+
+		for (const auto& sectorData : sectors)
+			write_sector(bits, cursor, sectorData);
+
+		if (cursor != bits.size())
+			Error() << "track data mismatched length";
+
+		std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+		fluxmap->appendBits(bits, clockRateUs * 1e3);
+		return fluxmap;
+	}
+
+private:
+	const MicropolisEncoderProto& _config;
+};
+
+std::unique_ptr<AbstractEncoder> createMicropolisEncoder(const EncoderProto& config)
+{
+	return std::unique_ptr<AbstractEncoder>(new MicropolisEncoder(config));
+}
+

arch/micropolis/micropolis.h

@@ -3,6 +3,14 @@
 
 #define MICROPOLIS_ENCODED_SECTOR_SIZE (1+2+266+6)
 
+class AbstractDecoder;
+class AbstractEncoder;
+class EncoderProto;
+class DecoderProto;
+
 extern std::unique_ptr<AbstractDecoder> createMicropolisDecoder(const DecoderProto& config);
+extern std::unique_ptr<AbstractEncoder> createMicropolisEncoder(const EncoderProto& config);
+
+extern uint8_t micropolisChecksum(const Bytes& bytes);
 
 #endif

arch/micropolis/micropolis.proto

@@ -1,4 +1,5 @@
 syntax = "proto2";
 
 message MicropolisDecoderProto {}
+message MicropolisEncoderProto {}
 

arch/mx/decoder.cc

@@ -31,7 +31,7 @@ public:
     void beginTrack()
 	{
 		_currentSector = -1;
-		_clock = 0;
+		_bitcell = 0;
 	}
 
     RecordType advanceToNextRecord()
@@ -40,8 +40,9 @@ public:
 		{
 			/* First sector in the track: look for the sync marker. */
 			const FluxMatcher* matcher = nullptr;
-			_sector->clock = _clock = _fmr->seekToPattern(ID_PATTERN, matcher);
-			readRawBits(32); /* skip the ID mark */
+			_sector->bitcell = _bitcell = _fmr->seekToPattern(ID_PATTERN, matcher);
+			_sector->clock = _sector->bitcell * 2;
+			 readRawBits(32); /* skip the ID mark */
 			_logicalTrack = decodeFmMfm(readRawBits(32)).slice(0, 32).reader().read_be16();
 		}
 		else if (_currentSector == 10)
@@ -54,7 +55,7 @@ public:
 			/* Otherwise we assume the clock from the first sector is still valid.
 			 * The decoder framwork will automatically stop when we hit the end of
 			 * the track. */
-			_sector->clock = _clock;
+			_sector->bitcell = _bitcell;
 		}
 
 		_currentSector++;
@@ -80,7 +81,7 @@ public:
 	}
 
 private:
-    nanoseconds_t _clock;
+    nanoseconds_t _bitcell;
     int _currentSector;
     int _logicalTrack;
 };

arch/northstar/decoder.cc

@@ -74,7 +74,7 @@ public:
 	{}
 
 	/* Search for FM or MFM sector record */
-	RecordType advanceToNextRecord()
+	RecordType advanceToNextRecord() override
 	{
 		nanoseconds_t now = _fmr->tell().ns();
 
@@ -105,7 +105,8 @@ public:
 		 * past one or more sector pulses.  For this reason, calculate
 		 * _hardSectorId after the sector header is found.
 		 */
-		_sector->clock = _fmr->seekToPattern(ANY_SECTOR_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(ANY_SECTOR_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 
 		int sectorFoundTimeRaw = std::round((_fmr->tell().ns()) / 1e6);
 		int sectorFoundTime;
@@ -140,7 +141,7 @@ public:
 		return UNKNOWN_RECORD;
 	}
 
-	void decodeSectorRecord()
+	void decodeSectorRecord() override
 	{
 		unsigned recordSize, payloadSize, headerSize;
 

arch/tids990/decoder.cc

@@ -48,7 +48,8 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return RecordType::SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)

arch/victor9k/decoder.cc

@@ -62,7 +62,7 @@ public:
     RecordType advanceToNextRecord()
 	{
 		const FluxMatcher* matcher = nullptr;
-		_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+		_sector->clock = _sector->bitcell = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 		if (matcher == &SECTOR_RECORD_PATTERN)
 			return SECTOR_RECORD;
 		if (matcher == &DATA_RECORD_PATTERN)
@@ -73,7 +73,7 @@ public:
     void decodeSectorRecord()
 	{
 		/* Skip the sync marker bit. */
-		readRawBits(23);
+		readRawBits(22);
 
 		/* Read header. */
 
@@ -96,7 +96,7 @@ public:
     void decodeDataRecord()
 	{
 		/* Skip the sync marker bit. */
-		readRawBits(23);
+		readRawBits(22);
 
 		/* Read data. */
 

arch/victor9k/encoder.cc

@@ -0,0 +1,200 @@
+#include "globals.h"
+#include "decoders/decoders.h"
+#include "encoders/encoders.h"
+#include "victor9k.h"
+#include "crc.h"
+#include "sector.h"
+#include "writer.h"
+#include "image.h"
+#include "fmt/format.h"
+#include "arch/victor9k/victor9k.pb.h"
+#include "lib/encoders/encoders.pb.h"
+#include <ctype.h>
+#include "bytes.h"
+
+static bool lastBit;
+
+static void write_zero_bits(std::vector<bool>& bits, unsigned& cursor, unsigned count)
+{
+    while (count--)
+	{
+		if (cursor < bits.size())
+			lastBit = bits[cursor++] = 0;
+	}
+}
+
+static void write_one_bits(std::vector<bool>& bits, unsigned& cursor, unsigned count)
+{
+    while (count--)
+	{
+		if (cursor < bits.size())
+			lastBit = bits[cursor++] = 1;
+	}
+}
+
+static void write_bits(std::vector<bool>& bits, unsigned& cursor, const std::vector<bool>& src)
+{
+	for (bool bit : src)
+	{
+		if (cursor < bits.size())
+			lastBit = bits[cursor++] = bit;
+	}
+}
+
+static void write_bits(std::vector<bool>& bits, unsigned& cursor, uint64_t data, int width)
+{
+	cursor += width;
+	lastBit = data & 1;
+	for (int i=0; i<width; i++)
+	{
+		unsigned pos = cursor - i - 1;
+		if (pos < bits.size())
+			bits[pos] = data & 1;
+		data >>= 1;
+	}
+}
+
+static void write_bits(std::vector<bool>& bits, unsigned& cursor, const Bytes& bytes)
+{
+	ByteReader br(bytes);
+	BitReader bitr(br);
+
+	while (!bitr.eof())
+	{
+		if (cursor < bits.size())
+			bits[cursor++] = bitr.get();
+	}
+}
+
+static int encode_data_gcr(uint8_t data)
+{
+    switch (data & 0x0f)
+    {
+        #define GCR_ENTRY(gcr, data) \
+            case data: return gcr;
+        #include "data_gcr.h"
+        #undef GCR_ENTRY
+    }
+    return -1;
+}
+
+static void write_bytes(std::vector<bool>& bits, unsigned& cursor, const Bytes& bytes)
+{
+    for (uint8_t b : bytes)
+    {
+        write_bits(bits, cursor, encode_data_gcr(b>>4), 5);
+        write_bits(bits, cursor, encode_data_gcr(b),    5);
+    }
+}
+
+static void write_sector(std::vector<bool>& bits, unsigned& cursor,
+		const Victor9kEncoderProto::TrackdataProto& trackdata,
+        const Sector& sector)
+{
+    write_one_bits(bits, cursor, trackdata.pre_header_sync_bits());
+    write_bits(bits, cursor, VICTOR9K_SECTOR_RECORD, 10);
+
+    uint8_t encodedTrack = sector.logicalTrack | (sector.logicalSide<<7);
+    uint8_t encodedSector = sector.logicalSector;
+    write_bytes(bits, cursor, Bytes {
+        encodedTrack,
+        encodedSector,
+        (uint8_t)(encodedTrack + encodedSector),
+    });
+
+
+    write_zero_bits(bits, cursor, trackdata.post_header_gap_bits());
+    write_one_bits(bits, cursor, trackdata.pre_data_sync_bits());
+    write_bits(bits, cursor, VICTOR9K_DATA_RECORD, 10);
+
+    write_bytes(bits, cursor, sector.data);
+
+    Bytes checksum(2);
+    checksum.writer().write_le16(sumBytes(sector.data));
+    write_bytes(bits, cursor, checksum);
+
+    write_zero_bits(bits, cursor, trackdata.post_data_gap_bits());
+}
+
+class Victor9kEncoder : public AbstractEncoder
+{
+public:
+	Victor9kEncoder(const EncoderProto& config):
+        AbstractEncoder(config),
+		_config(config.victor9k())
+	{}
+
+private:
+
+	void getTrackFormat(Victor9kEncoderProto::TrackdataProto& trackdata, unsigned cylinder, unsigned head)
+	{
+		trackdata.Clear();
+		for (const auto& f : _config.trackdata())
+		{
+			if (f.has_min_cylinder() && (cylinder < f.min_cylinder()))
+				continue;
+			if (f.has_max_cylinder() && (cylinder > f.max_cylinder()))
+				continue;
+			if (f.has_head() && (head != f.head()))
+				continue;
+
+			trackdata.MergeFrom(f);
+		}
+	}
+
+public:
+	std::vector<std::shared_ptr<Sector>> collectSectors(int physicalTrack, int physicalSide, const Image& image) override
+	{
+		std::vector<std::shared_ptr<Sector>> sectors;
+
+		Victor9kEncoderProto::TrackdataProto trackdata;
+		getTrackFormat(trackdata, physicalTrack, physicalSide);
+
+        for (int i = 0; i < trackdata.sector_range().sector_count(); i++)
+        {
+            int sectorId = trackdata.sector_range().start_sector() + i;
+			const auto& sector = image.get(physicalTrack, physicalSide, sectorId);
+			if (sector)
+				sectors.push_back(sector);
+        }
+
+		return sectors;
+	}
+
+    std::unique_ptr<Fluxmap> encode(int physicalTrack, int physicalSide,
+            const std::vector<std::shared_ptr<Sector>>& sectors, const Image& image) override
+    {
+		Victor9kEncoderProto::TrackdataProto trackdata;
+		getTrackFormat(trackdata, physicalTrack, physicalSide);
+
+        unsigned bitsPerRevolution = trackdata.original_data_rate_khz() * trackdata.original_period_ms();
+        std::vector<bool> bits(bitsPerRevolution);
+        double clockRateUs = 166666.0 / bitsPerRevolution;
+        unsigned cursor = 0;
+
+        fillBitmapTo(bits, cursor, trackdata.post_index_gap_us() / clockRateUs, { true, false });
+        lastBit = false;
+
+        for (const auto& sector : sectors)
+            write_sector(bits, cursor, trackdata, *sector);
+
+        if (cursor >= bits.size())
+            Error() << fmt::format("track data overrun by {} bits", cursor - bits.size());
+        fillBitmapTo(bits, cursor, bits.size(), { true, false });
+
+        std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+        fluxmap->appendBits(bits, clockRateUs*1e3);
+        return fluxmap;
+    }
+
+private:
+	const Victor9kEncoderProto& _config;
+};
+
+std::unique_ptr<AbstractEncoder> createVictor9kEncoder(const EncoderProto& config)
+{
+	return std::unique_ptr<AbstractEncoder>(new Victor9kEncoder(config));
+}
+
+// vim: sw=4 ts=4 et
+

arch/victor9k/victor9k.h

@@ -1,11 +1,22 @@
 #ifndef VICTOR9K_H
 #define VICTOR9K_H
 
-#define VICTOR9K_SECTOR_RECORD 0xfffffeab
-#define VICTOR9K_DATA_RECORD   0xfffffea4
+class AbstractEncoder;
+class AbstractDecoder;
+class EncoderProto;
+class DecoderProto;
+
+/* ... 1101 0101 0111
+ *       ^^ ^^^^ ^^^^ ten bit IO byte */
+#define VICTOR9K_SECTOR_RECORD 0xfffffd57 
+
+/* ... 1101 0100 1001
+ *       ^^ ^^^^ ^^^^ ten bit IO byte */
+#define VICTOR9K_DATA_RECORD   0xfffffd49
 
 #define VICTOR9K_SECTOR_LENGTH 512
 
 extern std::unique_ptr<AbstractDecoder> createVictor9kDecoder(const DecoderProto& config);
+extern std::unique_ptr<AbstractEncoder> createVictor9kEncoder(const EncoderProto& config);
 
 #endif

arch/victor9k/victor9k.proto

@@ -1,4 +1,32 @@
 syntax = "proto2";
 
+import "lib/common.proto";
+
 message Victor9kDecoderProto {}
 
+// NEXT: 12
+message Victor9kEncoderProto {
+	message TrackdataProto {
+		message SectorRangeProto {
+			optional int32 start_sector = 1 [(help) = "first sector ID on track"];
+			optional int32 sector_count = 2 [(help) = "number of sectors on track"];
+		}
+
+		optional int32 min_cylinder = 1            [(help) = "minimum cylinder this format applies to"];
+		optional int32 max_cylinder = 2            [(help) = "maximum cylinder this format applies to"];
+		optional int32 head = 3                    [(help) = "which head this format applies to"];
+
+		optional double original_period_ms = 4     [(help) = "original rotational period of this cylinder"];
+		optional double original_data_rate_khz = 5 [(help) = "original data rate of this cylinder"];
+		optional double post_index_gap_us = 6      [(help) = "size of post-index gap"];
+		optional int32 pre_header_sync_bits = 10   [(help) = "number of sync bits before the sector header"];
+		optional int32 pre_data_sync_bits = 8      [(help) = "number of sync bits before the sector data"];
+		optional int32 post_data_gap_bits = 9      [(help) = "size of gap between data and the next header"];
+		optional int32 post_header_gap_bits = 11   [(help) = "size of gap between header and the data"];
+
+		optional SectorRangeProto sector_range = 7 [(help) = "write these sectors on each track"];
+	}
+
+	repeated TrackdataProto trackdata = 1;
+}
+

arch/zilogmcz/decoder.cc

@@ -24,7 +24,8 @@ public:
 	{
 		const FluxMatcher* matcher = nullptr;
 		_fmr->seekToIndexMark();
-		_sector->clock = _fmr->seekToPattern(SECTOR_START_PATTERN, matcher);
+		_sector->bitcell = _fmr->seekToPattern(SECTOR_START_PATTERN, matcher);
+		_sector->clock = _sector->bitcell * 2;
 		if (matcher == &SECTOR_START_PATTERN)
 			return SECTOR_RECORD;
 		return UNKNOWN_RECORD;

doc/building.md

@@ -127,10 +127,11 @@ Programmer](https://www.cypress.com/products/psoc-programming-solutions).
 **Note:** _not_ the Cypress Programmer, which is for a different board!
 Cypress will make you register.
 
-Once done, run it. Plug the blunt end of the FluxEngine board into a USB
-port (the end which is a USB connector). The programmer should detect it
-and report it as a KitProg. You may be prompted to upgrade the programmer
-hardware; if so, follow the instructions and do it.
+Once done, run it. Plug the blunt end of the FluxEngine board into a USB port
+(the end which is a USB plug, with exposed traces; this is on the smaller
+section of the board). The programmer should detect it and report it as a
+KitProg. You may be prompted to upgrade the programmer hardware; if so, follow
+the instructions and do it.
 
 Now go to File -> File Load and open
 `FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex` in the
@@ -151,10 +152,11 @@ CY8CKIT-059 Kit Setup (Kit Design Files, Creator, Programmer, Documentation,
 Examples)'. I'm not linking to it in case the URL changes when they update
 it.
 
-Once this is done, I'd strongly recommend working through the initial
-tutorial and making the LED on your board flash. It'll tell you where all the
-controls are and how to program the board. Remember that the big end of the
-board plugs into your computer for programming.
+Once this is done, I'd strongly recommend working through the initial tutorial
+and making the LED on your board flash. It'll tell you where all the controls
+are and how to program the board. Remember that you have to plug the
+programming connector into your computer to flash it; the microusb socket is
+used only for application control.
 
 When you're ready, open the `FluxEngine.cydsn/FluxEngine.cyprj` project,
 pick 'Program' from the menu, and the firmware should compile and be

doc/disk-acorndfs.md

@@ -26,3 +26,9 @@ fluxengine read acorndfs
 You should end up with an `acorndfs.img` of the appropriate size for your disk
 format. This is an alias for `fluxengine read ibm` with preconfigured
 parameters.
+
+References
+----------
+
+  - [The Acord DFS disc format](https://beebwiki.mdfs.net/Acorn_DFS_disc_format)
+

doc/disk-ibm.md

@@ -14,7 +14,6 @@ metadata. Systems which use IBM scheme disks include but are not limited to:
   - the TRS-80
   - late era Commodore machines (the 1571 and so on)
   - most CP/M machines
-  - the Kaypro II
   - etc
 
 FluxEngine supports reading these. However, some variants are more peculiar
@@ -87,6 +86,10 @@ and there's too many configuration options to usefully list. Use `fluxengine
 write` to list all formats, and try `fluxengine write ibm1440 --config` to see
 a sample configuration.
 
+Some image formats, such as DIM, specify the image format, For these you can
+specify the `ibm` format and FluxEngine will automatically determine the
+correct format to use.
+
 Mixed-format disks
 ------------------
 

doc/disk-micropolis.md

@@ -37,6 +37,15 @@ could be represented as having either twice the number of sectors, for CHS, or
 twice the number of tracks, HCS; the second side's tracks logically followed
 the first side (e.g., tracks 77-153). Micropolis disks tended to be the latter.
 
+Writing disks
+-------------
+
+Just do:
+
+```
+fluxengine write micropolis -i micropolis.img
+```
+
 Useful references
 -----------------
 

doc/disk-victor9k.md

@@ -6,9 +6,28 @@ which used a disk format very reminiscent of the Commodore format; not a
 coincidence, as Chuck Peddle designed them both. They're 80-track, 512-byte
 sector GCR disks, with a variable-speed drive and a varying number of sectors
 per track --- from 19 to 12. Disks can be double-sided, meaning that they can
-store 1224kB per disk, which was almost unheard of back then.
+store 1224kB per disk, which was almost unheard of back then. Because the way
+that the tracks on head 1 are offset from head 0 (this happens with all disks),
+the speed zone allocation on head 1 differ from head 0...
 
-FluxEngine reads these.
+| Zone | Head 0 tracks | Head 1 tracks | Sectors | Original period (ms) |
+|:----:|:-------------:|:-------------:|:-------:|:--------------------:|
+| 0    | 0-3           |               | 19      | 237.9                |
+| 1    | 4-15          | 0-7           | 18      | 224.5                |
+| 2    | 16-26         | 8-18          | 17      | 212.2                |
+| 3    | 27-37         | 19-29         | 16      | 199.9                |
+| 4    | 38-48         | 30-40         | 15      | 187.6                |
+| 5    | 49-59         | 41-51         | 14      | 175.3                |
+| 6    | 60-70         | 52-62         | 13      | 163.0                |
+| 7    | 71-79         | 63-74         | 12      | 149.6                |
+| 8    |               | 75-79         | 11      | 144.0                |
+
+(The Original Period column is the original rotation rate. When used in
+FluxEngine, the disk always spins at 360 rpm, which corresponds to a rotational
+period of 166 ms.)
+
+FluxEngine can read and write the single-sided variant of these. (Double-sided
+will be trivial to do, it's just not done yet.)
 
 Reading discs
 -------------
@@ -16,18 +35,25 @@ Reading discs
 Just do:
 
 ```
-fluxengine read victor9k
+fluxengine read victor9k-ss
 ```
 
-You should end up with an `victor9k.img` which is 774656 bytes long.
-if you want the double-sided variety, use `--heads 0-1`.
+You should end up with an `victor9k.img` which is 627200 bytes long.
 
-**Big warning!** The image may not work in an emulator. Victor disk images are
-complicated due to the way the tracks are different sizes and the odd sector
-size. FluxEngine chooses to store them in a simple 512 x 19 x 1 x 80 layout,
-with holes where missing sectors should be. This was easiest. If anyone can
-suggest a better way, please [get in
-touch](https://github.com/davidgiven/fluxengine/issues/new).
+**Big warning!** The image is triangular, where each track occupies a different
+amount of space. Victor disk images are complicated due to the way the tracks
+are different sizes and the odd sector size.
+
+Writing discs
+-------------
+
+Just do:
+
+```
+fluxengine read victor9k-ss -i victor9k.img
+```
+
+**Big warning!** This uses the same triangular disk image that reading uses.
 
 
 Useful references

doc/using.md

@@ -233,7 +233,7 @@ FluxEngine supports a number of ways to get or put flux. When using the `-s` or
 FluxEngine also supports a number of file system image formats. When using the
 `-i` or `-o` options (for input and output), you can use any of these strings:
 
-  - `<filename.adf>`, `<filename.d81>`, `<filename.img>`, `<filename.st>`
+  - `<filename.adf>`, `<filename.d81>`, `<filename.img>`, `<filename.st>`, `<filename.xdf>`
 
 	Read from or write to a simple headerless image file (all these formats are
 	the same). This will probably want configuration via the
@@ -246,10 +246,38 @@ FluxEngine also supports a number of file system image formats. When using the
 	4.2](https://en.wikipedia.org/wiki/Disk_Copy) image file, commonly used by
 	Apple Macintosh emulators.
   
+  - `<filename.td0>`
+
+	Read a [Sydex Teledisk TD0
+	file](https://web.archive.org/web/20210420224336/http://dunfield.classiccmp.org/img47321/teledisk.htm)
+	image file. Note that only uncompressed images are supported (so far).
+
   - `<filename.jv3>`
 
 	Read from a JV3 image file, commonly used by TRS-80 emulators. **Read
 	only.**
+
+  - `<filename.dim>`
+
+  Read from a [DIM image file](https://www.pc98.org/project/doc/dim.html),
+  commonly used by X68000 emulators. Supports automatically configuring
+  the encoder. **Read Only.**
+  
+  - `<filename.fdi>`
+
+  Read from a [FDI image file](https://www.pc98.org/project/doc/hdi.html),
+  commonly used by PC-98 emulators. **Read Only.**
+  
+  - `<filename.d88>`
+
+  Read from a [D88 image file](https://www.pc98.org/project/doc/d88.html),
+  commonly used by various Japanese PC emulators, including the NEC PC-88. **Read Only.**
+  
+  FluxEngine is currently limited to reading only the first floppy image in a
+  D88 file.
+  
+  The D88 reader should be used with the `ibm` profile and will override
+  most encoding parameters on a track-by-track basis.
   
   - `<filename.ldbs>`
 
@@ -269,7 +297,7 @@ disks, and have different magnetic properties. 3.5" drives can usually
 autodetect what kind of medium is inserted into the drive based on the hole in
 the disk casing, but 5.25" drives can't. As a result, you need to explicitly
 tell FluxEngine on the command line whether you're using a high density disk or
-not with the `--input/output.flux.drive.high_density` configuration setting.
+not with the `--flux_source/sink.drive.high_density` configuration setting.
 **If you don't do this, your disks may not read correctly and will _certainly_
 fail to write correctly.**
 
@@ -286,14 +314,14 @@ here.](http://www.retrotechnology.com/herbs_stuff/guzis.html)
 These flags apply to many operations and are useful for modifying the overall
 behaviour.
 
-  - `--input.flux.drive.revolutions=X`
+  - `--flux_source.drive.revolutions=X`
 
     When reading, spin the disk X times. X
 	can be a floating point number. The default is usually 1.2. Some formats
 	default to 1.  Increasing the number will sample more data, and can be
 	useful on dubious disks to try and get a better read.
 
-  - `--input.flux.drive.sync_with_index=true|false`
+  - `--flux_source.drive.sync_with_index=true|false`
 
     Wait for an index pulse
 	before starting to read the disk. (Ignored for write operations.) By
@@ -301,7 +329,7 @@ behaviour.
 	disk problems it's helpful to have all your data start at the same place
 	each time.
 
-  - `--input.flux.drive.index_source=X`, `--output.flux.drive.index_source=X`
+  - `--flux_source.drive.index_source=X`, `--flux_sink.drive.index_source=X`
 
 	Set the source of index pulses when reading or writing respectively. This
 	is for use with drives which don't produce index pulse data. `X` can be

lib/bytes.cc

@@ -141,7 +141,7 @@ Bytes Bytes::slice(unsigned start, unsigned len) const
     {
         /* Can't share the buffer, as we need to zero-pad the end. */
         Bytes b(len);
-        std::uninitialized_copy(cbegin()+start, cend(), b.begin());
+        std::uninitialized_copy(_data->cbegin()+start, _data->cbegin()+_high, b._data->begin());
         return b;
     }
     else

lib/bytes.h

@@ -41,7 +41,7 @@ public:
     uint8_t* begin()              { checkWritable(); return &(*_data)[_low]; }
     uint8_t* end()                { checkWritable(); return &(*_data)[_high]; }
 
-	operator const std::string () const { return std::string(cbegin(), cend()); }
+	operator std::string () const { return std::string(cbegin(), cend()); }
 
     void boundsCheck(unsigned pos) const;
     void checkWritable();

lib/data.proto

@@ -1,40 +0,0 @@
-syntax = "proto2";
-
-// Images
-
-enum SectorStatus {
-	UNKNOWN = 0;
-	OK = 1;
-	BAD_CHECKSUM = 2;
-	MISSING = 3;
-	DATA_MISSING = 4;
-	CONFLICT = 5;
-	INTERNAL_ERROR = 6;
-}
-
-message SectorProto {
-	optional int32 logical_sector = 1;
-	optional bytes data = 2;
-	optional SectorStatus status = 3;
-
-    optional uint64 clock = 4;
-    optional uint64 header_starttime_ns = 5;
-    optional uint64 header_endtime_ns = 6;
-    optional uint64 data_starttime_ns = 7;
-    optional uint64 data_endtime_ns = 8;
-    optional int32 physical_cylinder = 9;
-    optional int32 physical_head = 10;
-    optional int32 logical_track = 11;
-    optional int32 logical_side = 12;
-}
-
-message TrackProto {
-	map<int32, SectorProto> sectors = 1;
-	optional int32 logical_track = 2;
-	optional int32 logical_side = 3;
-}
-
-message ImageProto {
-	map<int32, TrackProto> tracks = 1;
-}
-

lib/decoders/decoders.cc

@@ -25,7 +25,6 @@
 #include "sector.h"
 #include "image.h"
 #include "lib/decoders/decoders.pb.h"
-#include "lib/data.pb.h"
 #include "fmt/format.h"
 #include <numeric>
 
@@ -78,18 +77,20 @@ std::unique_ptr<TrackDataFlux> AbstractDecoder::decodeToSectors(
 		_sector->physicalHead = physicalHead;
 
         Fluxmap::Position recordStart = fmr.tell();
+		_decoder.reset(new FluxDecoder(&fmr, _sector->bitcell, _config));
         RecordType r = advanceToNextRecord();
-        if (fmr.eof() || !_sector->clock)
+        if (fmr.eof() || !_sector->bitcell)
             return std::move(_trackdata);
         if ((r == UNKNOWN_RECORD) || (r == DATA_RECORD))
         {
-            fmr.findEvent(F_BIT_PULSE);
+            fmr.skipToEvent(F_BIT_PULSE);
             continue;
         }
 
         /* Read the sector record. */
 
         recordStart = fmr.tell();
+		resetFluxDecoder();
         decodeSectorRecord();
         Fluxmap::Position recordEnd = fmr.tell();
         pushRecord(recordStart, recordEnd);
@@ -104,12 +105,15 @@ std::unique_ptr<TrackDataFlux> AbstractDecoder::decodeToSectors(
 				r = advanceToNextRecord();
 				if (r != UNKNOWN_RECORD)
 					break;
-				if (fmr.findEvent(F_BIT_PULSE) == 0)
+				if (fmr.eof())
                     break;
 			}
             recordStart = fmr.tell();
             if (r == DATA_RECORD)
+			{
+				resetFluxDecoder();
                 decodeDataRecord();
+			}
             recordEnd = fmr.tell();
             pushRecord(recordStart, recordEnd);
         }
@@ -130,16 +134,22 @@ void AbstractDecoder::pushRecord(const Fluxmap::Position& start, const Fluxmap::
 	
 	record->startTime = start.ns();
 	record->endTime = end.ns();
-    record->clock = _sector->clock;
+    record->bitcell = _sector->bitcell;
 
     _fmr->seek(start);
-    record->rawData = toBytes(_fmr->readRawBits(end, _sector->clock));
+	FluxDecoder decoder(_fmr, _sector->bitcell, _config);
+    record->rawData = toBytes(decoder.readBits(end));
     _fmr->seek(here);
 }
 
+void AbstractDecoder::resetFluxDecoder()
+{
+	_decoder.reset(new FluxDecoder(_fmr, _sector->bitcell, _config));
+}
+
 std::vector<bool> AbstractDecoder::readRawBits(unsigned count)
 {
-	return _fmr->readRawBits(count, _sector->clock);
+	return _decoder->readBits(count);
 }
 
 std::set<unsigned> AbstractDecoder::requiredSectors(unsigned cylinder, unsigned head) const

lib/decoders/decoders.h

@@ -4,6 +4,7 @@
 #include "bytes.h"
 #include "sector.h"
 #include "decoders/fluxmapreader.h"
+#include "decoders/fluxdecoder.h"
 
 class Sector;
 class Fluxmap;
@@ -47,8 +48,8 @@ public:
     std::unique_ptr<TrackDataFlux> decodeToSectors(std::shared_ptr<const Fluxmap> fluxmap, unsigned cylinder, unsigned head);
     void pushRecord(const Fluxmap::Position& start, const Fluxmap::Position& end);
 
+	void resetFluxDecoder();
     std::vector<bool> readRawBits(unsigned count);
-    //{ return _fmr->readRawBits(count, _sector->clock); }
 
     Fluxmap::Position tell()
     { return _fmr->tell(); } 
@@ -68,6 +69,7 @@ protected:
     FluxmapReader* _fmr = nullptr;
 	std::unique_ptr<TrackDataFlux> _trackdata;
     std::shared_ptr<Sector> _sector;
+	std::unique_ptr<FluxDecoder> _decoder;
 };
 
 #endif

lib/decoders/decoders.proto

@@ -18,16 +18,19 @@ import "arch/zilogmcz/zilogmcz.proto";
 import "lib/fluxsink/fluxsink.proto";
 import "lib/common.proto";
 
+//NEXT: 27
 message DecoderProto {
 	optional double pulse_debounce_threshold = 1 [default = 0.30,
 		(help) = "ignore pulses with intervals shorter than this, in fractions of a clock"];
 	optional double bit_error_threshold = 2 [default = 0.40,
 		(help) = "amount of error to tolerate in pulse timing, in fractions of a clock"];
-	optional double clock_interval_bias = 3 [default = -0.02,
-		(help) = "adjust intervals between pulses by this many clocks before decoding"];
 	optional double minimum_clock_us = 4 [default = 0.75,
 		(help) = "refuse to detect clocks shorter than this, to avoid false positives"];
 
+	optional double pll_adjust = 25 [default = 0.04];
+	optional double pll_phase = 26 [default = 0.60];
+	optional double flux_scale = 27 [default = 1.0];
+
 	oneof format {
 		IbmDecoderProto ibm = 5;
 		BrotherDecoderProto brother = 6;

lib/decoders/fluxdecoder.cc

@@ -0,0 +1,117 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "decoders/fluxdecoder.h"
+#include "lib/decoders/decoders.pb.h"
+
+/* This is a port of the samdisk code:
+ *
+ * https://github.com/simonowen/samdisk/blob/master/src/FluxDecoder.cpp
+ *
+ * I'm not actually terribly sure how it works, but it does, and much better
+ * than my code.
+ */
+
+FluxDecoder::FluxDecoder(FluxmapReader* fmr, nanoseconds_t bitcell,
+		const DecoderProto& config):
+	_fmr(fmr),
+	_pll_phase(config.pll_phase()),
+	_pll_adjust(config.pll_adjust()),
+	_flux_scale(config.flux_scale()),
+	_clock(bitcell),
+	_clock_centre(bitcell),
+	_clock_min(bitcell * (1.0 - _pll_adjust)),
+	_clock_max(bitcell * (1.0 + _pll_adjust)),
+	_flux(0),
+	_leading_zeroes(fmr->tell().zeroes)
+{}
+
+
+bool FluxDecoder::readBit()
+{
+	if (_leading_zeroes > 0)
+	{
+		_leading_zeroes--;
+		return false;
+	}
+	else if (_leading_zeroes == 0)
+	{
+		_leading_zeroes--;
+		return true;
+	}
+
+	while (!_fmr->eof() && (_flux < (_clock/2)))
+	{
+		_flux += nextFlux() * _flux_scale;;
+		_clocked_zeroes = 0;
+	}
+
+	_flux -= _clock;
+	if (_flux >= (_clock/2))
+	{
+		_clocked_zeroes++;
+		_goodbits++;
+		return false;
+	}
+
+	/* PLL adjustment: change the clock frequency according to the phase
+	 * mismatch */
+	if (_clocked_zeroes <= 3)
+	{
+		/* In sync: adjust base clock */
+		
+		_clock += _flux * _pll_adjust;
+	}
+	else
+	{
+		/* Out of sync: adjust the base clock back towards the centre */
+
+		_clock += (_clock_centre - _clock) * _pll_adjust;
+
+		/* We require 256 good bits before reporting another sync loss event. */
+
+		if (_goodbits >= 256)
+			_sync_lost = true;
+		_goodbits = 0;
+	}
+
+	/* Clamp the clock's adjustment range. */
+
+	_clock = std::min(std::max(_clock_min, _clock), _clock_max);
+
+	/* I'm not sure what this does, but the original comment is:
+     * Authentic PLL: Do not snap the timing window to each flux transition
+	 */
+
+    _flux = _flux * (1.0 - _pll_phase);
+
+	_goodbits++;
+	return true;
+}
+
+std::vector<bool> FluxDecoder::readBits(unsigned count)
+{
+    std::vector<bool> result;
+    while (!_fmr->eof() && count--)
+    {
+        bool b = readBit();
+        result.push_back(b);
+    }
+    return result;
+}
+
+std::vector<bool> FluxDecoder::readBits(const Fluxmap::Position& until)
+{
+    std::vector<bool> result;
+    while (!_fmr->eof() && (_fmr->tell().bytes < until.bytes))
+    {
+        bool b = readBit();
+        result.push_back(b);
+    }
+    return result;
+}
+
+nanoseconds_t FluxDecoder::nextFlux()
+{
+	return _fmr->readInterval(_clock_centre) * NS_PER_TICK;
+}

lib/decoders/fluxdecoder.h

@@ -0,0 +1,37 @@
+#ifndef FLUXDECODER_H
+#define FLUXDECODER_H
+
+class FluxmapReader;
+
+class FluxDecoder
+{
+public:
+	FluxDecoder(FluxmapReader* fmr, nanoseconds_t bitcell,
+		const DecoderProto& config);
+
+	bool readBit();
+    std::vector<bool> readBits(unsigned count);
+    std::vector<bool> readBits(const Fluxmap::Position& until);
+
+private:
+	nanoseconds_t nextFlux();
+
+private:
+	FluxmapReader* _fmr;
+	double _pll_phase;
+	double _pll_adjust;
+	double _flux_scale;
+	nanoseconds_t _clock = 0;
+	nanoseconds_t _clock_centre;
+	nanoseconds_t _clock_min;
+	nanoseconds_t _clock_max;
+	nanoseconds_t _flux = 0;
+	unsigned _clocked_zeroes = 0;
+	unsigned _goodbits = 0;
+	bool _index = false;
+	bool _sync_lost = false;
+	int _leading_zeroes;
+};
+
+#endif
+

lib/decoders/fluxmapreader.cc

@@ -18,7 +18,7 @@ FluxmapReader::FluxmapReader(const Fluxmap& fluxmap):
 	rewind();
 }
 
-uint8_t FluxmapReader::getNextEvent(unsigned& ticks)
+void FluxmapReader::getNextEvent(int& event, unsigned& ticks)
 {
     ticks = 0;
 
@@ -26,30 +26,42 @@ uint8_t FluxmapReader::getNextEvent(unsigned& ticks)
     {
         uint8_t b = _bytes[_pos.bytes++];
 		ticks += b & 0x3f;
-		if (b & (F_BIT_PULSE|F_BIT_INDEX))
+		if (!b || (b & (F_BIT_PULSE|F_BIT_INDEX)))
         {
             _pos.ticks += ticks;
-            return b;
+			event = b & 0xc0;
+            return;
         }
     }
 
     _pos.ticks += ticks;
-    return 0;
+	event = F_EOF;
 }
 
-unsigned FluxmapReader::findEvent(uint8_t target)
+void FluxmapReader::skipToEvent(int event)
 {
-    unsigned ticks = 0;
+	unsigned ticks;
+	findEvent(event, ticks);
+}
+
+bool FluxmapReader::findEvent(int event, unsigned& ticks)
+{
+    ticks = 0;
 
     for (;;)
     {
         unsigned thisTicks;
-        uint8_t bits = getNextEvent(thisTicks);
-        ticks += thisTicks;
+		int thisEvent;
+		getNextEvent(thisEvent, thisTicks);
+		ticks += thisTicks;
+
+		if (thisEvent == F_EOF)
+			return false;
+
         if (eof())
-            return 0;
-        if (bits & target)
-            return ticks;
+            return false;
+		if ((event == thisEvent) || (event & thisEvent))
+			return true;
     }
 }
 
@@ -58,10 +70,10 @@ unsigned FluxmapReader::readInterval(nanoseconds_t clock)
     unsigned thresholdTicks = (clock * _config.pulse_debounce_threshold()) / NS_PER_TICK;
     unsigned ticks = 0;
 
-    while (ticks < thresholdTicks)
+    while (ticks <= thresholdTicks)
     {
-        unsigned thisTicks = findEvent(F_BIT_PULSE);
-        if (!thisTicks)
+        unsigned thisTicks;
+		if (!findEvent(F_BIT_PULSE, thisTicks))
             break;
         ticks += thisTicks;
     }
@@ -183,8 +195,9 @@ void FluxmapReader::seek(nanoseconds_t ns)
 
     while (!eof() && (_pos.ticks < ticks))
     {
+		int e;
         unsigned t;
-        getNextEvent(t);
+        getNextEvent(e, t);
     }
     _pos.zeroes = 0;
 }
@@ -225,7 +238,7 @@ nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern, const Flu
             positions[i] = positions[i+1];
             candidates[i] = candidates[i+1];
         }
-        candidates[intervalCount] = findEvent(F_BIT_PULSE);
+		findEvent(F_BIT_PULSE, candidates[intervalCount]);
         positions[intervalCount] = tell();
 
     }
@@ -236,44 +249,7 @@ nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern, const Flu
 
 void FluxmapReader::seekToIndexMark()
 {
-    findEvent(F_BIT_INDEX);
+    skipToEvent(F_BIT_INDEX);
     _pos.zeroes = 0;
 }
 
-bool FluxmapReader::readRawBit(nanoseconds_t clockPeriod)
-{
-    assert(clockPeriod != 0);
-
-    if (_pos.zeroes)
-    {
-        _pos.zeroes--;
-        return false;
-    }
-
-    nanoseconds_t interval = readInterval(clockPeriod)*NS_PER_TICK;
-    double clocks = (double)interval / clockPeriod + _config.clock_interval_bias();
-
-    if (clocks < 1.0)
-        clocks = 1.0;
-    _pos.zeroes = (int)round(clocks) - 1;
-    return true;
-}
-
-std::vector<bool> FluxmapReader::readRawBits(unsigned count, nanoseconds_t clockPeriod)
-{
-    std::vector<bool> result;
-    while (!eof() && count--)
-    {
-        bool b = readRawBit(clockPeriod);
-        result.push_back(b);
-    }
-    return result;
-}
-
-std::vector<bool> FluxmapReader::readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod)
-{
-    std::vector<bool> result;
-    while (!eof() && (_pos.bytes < until.bytes))
-        result.push_back(readRawBit(clockPeriod));
-    return result;
-}

lib/decoders/fluxmapreader.h

@@ -93,8 +93,9 @@ public:
         return (_fluxmap.duration());
     }
 
-    uint8_t getNextEvent(unsigned& ticks);
-    unsigned findEvent(uint8_t bits);
+    void getNextEvent(int& event, unsigned& ticks);
+	void skipToEvent(int event);
+    bool findEvent(int event, unsigned& ticks);
     unsigned readInterval(nanoseconds_t clock); /* with debounce support */
 
     /* Important! You can only reliably seek to 1 bits. */
@@ -104,10 +105,6 @@ public:
     nanoseconds_t seekToPattern(const FluxMatcher& pattern);
     nanoseconds_t seekToPattern(const FluxMatcher& pattern, const FluxMatcher*& matching);
 
-    bool readRawBit(nanoseconds_t clockPeriod);
-    std::vector<bool> readRawBits(unsigned count, nanoseconds_t clockPeriod);
-    std::vector<bool> readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod);
-
 private:
     const Fluxmap& _fluxmap;
     const uint8_t* _bytes;

lib/encoders/encoders.cc

@@ -7,8 +7,10 @@
 #include "arch/c64/c64.h"
 #include "arch/ibm/ibm.h"
 #include "arch/macintosh/macintosh.h"
+#include "arch/micropolis/micropolis.h"
 #include "arch/northstar/northstar.h"
 #include "arch/tids990/tids990.h"
+#include "arch/victor9k/victor9k.h"
 #include "lib/encoders/encoders.pb.h"
 #include "protocol.h"
 
@@ -22,8 +24,10 @@ std::unique_ptr<AbstractEncoder> AbstractEncoder::create(const EncoderProto& con
 		{ EncoderProto::kC64,       createCommodore64Encoder },
 		{ EncoderProto::kIbm,       createIbmEncoder },
 		{ EncoderProto::kMacintosh, createMacintoshEncoder },
+		{ EncoderProto::kMicropolis,createMicropolisEncoder },
 		{ EncoderProto::kNorthstar, createNorthstarEncoder },
 		{ EncoderProto::kTids990,   createTids990Encoder },
+		{ EncoderProto::kVictor9K,  createVictor9kEncoder },
 	};
 
 	auto encoder = encoders.find(config.format_case());

lib/encoders/encoders.proto

@@ -5,8 +5,10 @@ import "arch/brother/brother.proto";
 import "arch/c64/c64.proto";
 import "arch/ibm/ibm.proto";
 import "arch/macintosh/macintosh.proto";
+import "arch/micropolis/micropolis.proto";
 import "arch/northstar/northstar.proto";
 import "arch/tids990/tids990.proto";
+import "arch/victor9k/victor9k.proto";
 //import "lib/common.proto";
 
 message EncoderProto {
@@ -18,5 +20,7 @@ message EncoderProto {
 		Tids990EncoderProto tids990 = 7;
 		Commodore64EncoderProto c64 = 8;
 		NorthstarEncoderProto northstar = 9;
+		MicropolisEncoderProto micropolis = 10;
+		Victor9kEncoderProto victor9k = 11;
 	}
 }

lib/fl2.proto

@@ -0,0 +1,22 @@
+syntax = "proto2";
+
+enum FluxMagic {
+	MAGIC = 0x466c7578;
+}
+
+enum FluxFileVersion {
+	VERSION_1 = 1;
+}
+
+message TrackFluxProto {
+	optional int32 cylinder = 1;
+	optional int32 head = 2;
+	optional bytes flux = 3;
+}
+
+message FluxFileProto {
+	optional int32 magic = 1;
+	optional FluxFileVersion version = 2;
+	repeated TrackFluxProto track = 3;
+}
+

lib/flux.h

@@ -6,7 +6,7 @@ class Image;
 
 struct Record
 {
-	nanoseconds_t clock = 0;
+	nanoseconds_t bitcell = 0;
 	nanoseconds_t startTime = 0;
 	nanoseconds_t endTime = 0;
 	Bytes rawData;

lib/fluxmap.cc

@@ -56,6 +56,12 @@ Fluxmap& Fluxmap::appendIndex()
     return *this;
 }
 
+Fluxmap& Fluxmap::appendDesync()
+{
+	appendByte(F_DESYNC);
+    return *this;
+}
+
 void Fluxmap::precompensate(int threshold_ticks, int amount_ticks)
 {
     uint8_t junk = 0xff;

lib/fluxmap.h

@@ -46,6 +46,7 @@ public:
     Fluxmap& appendInterval(uint32_t ticks);
     Fluxmap& appendPulse();
     Fluxmap& appendIndex();
+	Fluxmap& appendDesync();
 
     Fluxmap& appendBytes(const Bytes& bytes);
     Fluxmap& appendBytes(const uint8_t* ptr, size_t len);

lib/fluxsink/aufluxsink.cc

@@ -68,14 +68,15 @@ public:
 			while (!fmr.eof())
 			{
 				unsigned ticks;
-				uint8_t bits = fmr.getNextEvent(ticks);
+				int event;
+				fmr.getNextEvent(event, ticks);
 				if (fmr.eof())
 					break;
 				timestamp += ticks;
 
-				if (bits & F_BIT_PULSE)
+				if (event & F_BIT_PULSE)
 					data[timestamp*channels + 0] = 0x7f;
-				if (_config.index_markers() && (bits & F_BIT_INDEX))
+				if (_config.index_markers() && (event & F_BIT_INDEX))
 					data[timestamp*channels + 1] = 0x7f;
 			}
 

lib/fluxsink/fl2fluxsink.cc

@@ -0,0 +1,70 @@
+#include "globals.h"
+#include "flags.h"
+#include "fluxmap.h"
+#include "sql.h"
+#include "bytes.h"
+#include "protocol.h"
+#include "fluxsink/fluxsink.h"
+#include "decoders/fluxmapreader.h"
+#include "lib/fluxsink/fluxsink.pb.h"
+#include "proto.h"
+#include "fmt/format.h"
+#include "lib/fl2.pb.h"
+#include <fstream>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+class Fl2FluxSink : public FluxSink
+{
+public:
+	Fl2FluxSink(const Fl2FluxSinkProto& lconfig):
+		_config(lconfig),
+		_of(lconfig.filename(), std::ios::out | std::ios::binary)
+	{
+		if (!_of.is_open())
+			Error() << "cannot open output file";
+	}
+
+	~Fl2FluxSink()
+	{
+		FluxFileProto proto;
+		proto.set_magic(FluxMagic::MAGIC);
+		proto.set_version(FluxFileVersion::VERSION_1);
+		for (const auto& e : _data)
+		{
+			auto track = proto.add_track();
+			track->set_cylinder(e.first.first);
+			track->set_head(e.first.second);
+			track->set_flux(e.second);
+		}
+
+		if (!proto.SerializeToOstream(&_of))
+			Error() << "unable to write output file";
+		_of.close();
+		if (_of.fail())
+			Error() << "FL2 write I/O error: " << strerror(errno);
+	}
+
+public:
+	void writeFlux(int cylinder, int head, Fluxmap& fluxmap)
+	{
+		_data[std::make_pair(cylinder, head)] = fluxmap.rawBytes();
+	}
+
+	operator std::string () const
+	{
+		return fmt::format("fl2({})", _config.filename());
+	}
+
+private:
+	const Fl2FluxSinkProto& _config;
+	std::ofstream _of;
+	std::map<std::pair<unsigned, unsigned>, Bytes> _data;
+};
+
+std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(const Fl2FluxSinkProto& config)
+{
+    return std::unique_ptr<FluxSink>(new Fl2FluxSink(config));
+}
+
+

lib/fluxsink/fluxsink.cc

@@ -26,6 +26,9 @@ std::unique_ptr<FluxSink> FluxSink::create(const FluxSinkProto& config)
 		case FluxSinkProto::kScp:
 			return createScpFluxSink(config.scp());
 
+		case FluxSinkProto::kFl2:
+			return createFl2FluxSink(config.fl2());
+
 		default:
 			Error() << "bad output disk config";
 			return std::unique_ptr<FluxSink>();
@@ -36,7 +39,7 @@ void FluxSink::updateConfigForFilename(FluxSinkProto* proto, const std::string&
 {
 	static const std::vector<std::pair<std::regex, std::function<void(const std::string&)>>> formats =
 	{
-		{ std::regex("^(.*\\.flux)$"), [&](const auto& s) { proto->set_fluxfile(s); }},
+		{ std::regex("^(.*\\.flux)$"), [&](const auto& s) { proto->mutable_fl2()->set_filename(s); }},
 		{ std::regex("^(.*\\.scp)$"),  [&](const auto& s) { proto->mutable_scp()->set_filename(s); }},
 		{ std::regex("^vcd:(.*)$"),    [&](const auto& s) { proto->mutable_vcd()->set_directory(s); }},
 		{ std::regex("^au:(.*)$"),     [&](const auto& s) { proto->mutable_au()->set_directory(s); }},

lib/fluxsink/fluxsink.h

@@ -10,6 +10,7 @@ class HardwareFluxSinkProto;
 class AuFluxSinkProto;
 class VcdFluxSinkProto;
 class ScpFluxSinkProto;
+class Fl2FluxSinkProto;
 
 class FluxSink
 {
@@ -21,6 +22,7 @@ public:
     static std::unique_ptr<FluxSink> createAuFluxSink(const AuFluxSinkProto& config);
     static std::unique_ptr<FluxSink> createVcdFluxSink(const VcdFluxSinkProto& config);
     static std::unique_ptr<FluxSink> createScpFluxSink(const ScpFluxSinkProto& config);
+    static std::unique_ptr<FluxSink> createFl2FluxSink(const Fl2FluxSinkProto& config);
 
     static std::unique_ptr<FluxSink> create(const FluxSinkProto& config);
 	static void updateConfigForFilename(FluxSinkProto* proto, const std::string& filename);

lib/fluxsink/fluxsink.proto

@@ -24,6 +24,10 @@ message ScpFluxSinkProto {
 	optional int32 type_byte = 4       [default = 0xff, (help) = "set the SCP disk type byte"];
 }
 
+message Fl2FluxSinkProto {
+	optional string filename = 1       [default = "flux.fl2", (help) = ".fl2 file to write to"];
+}
+
 message FluxSinkProto {
 	oneof dest {
 		string fluxfile = 1 [(help) = "name of destination flux file"];
@@ -31,6 +35,7 @@ message FluxSinkProto {
 		AuFluxSinkProto au = 3;
 		VcdFluxSinkProto vcd = 4;
 		ScpFluxSinkProto scp = 5;
+		Fl2FluxSinkProto fl2 = 6;
 	}
 }
 

lib/fluxsink/scpfluxsink.cc

@@ -47,7 +47,6 @@ public:
 		_fileheader.file_id[2] = 'P';
 		_fileheader.version = 0x18; /* Version 1.8 of the spec */
 		_fileheader.type = _config.type_byte();
-		_fileheader.revolutions = 5;
 		_fileheader.start_track = strackno(config.cylinders().start(), config.heads().start());
 		_fileheader.end_track = strackno(config.cylinders().end(), config.heads().end());
 		_fileheader.flags = (_config.align_with_index() ? SCP_FLAG_INDEXED : 0)
@@ -55,7 +54,7 @@ public:
 		_fileheader.cell_width = 0;
 		_fileheader.heads = singlesided;
 
-		std::cout << fmt::format("Writing 96 tpi {} SCP file containing {} SCP tracks\n",
+		std::cout << fmt::format("SCP: writing 96 tpi {} file containing {} tracks\n",
 			singlesided ? "single sided" : "double sided",
 			_fileheader.end_track - _fileheader.start_track + 1
 		);
@@ -71,7 +70,7 @@ public:
 		appendChecksum(checksum, _trackdata);
 		write_le32(_fileheader.checksum, checksum);
 
-		std::cout << "Writing output file...\n";
+		std::cout << "SCP: writing output file...\n";
 		std::ofstream of(_config.filename(), std::ios::out | std::ios::binary);
 		if (!of.is_open())
 			Error() << "cannot open output file";
@@ -88,17 +87,17 @@ public:
 		int strack = strackno(cylinder, head);
 
 		ScpTrack trackheader = {0};
-		trackheader.track_id[0] = 'T';
-		trackheader.track_id[1] = 'R';
-		trackheader.track_id[2] = 'K';
-		trackheader.strack = strack;
+		trackheader.header.track_id[0] = 'T';
+		trackheader.header.track_id[1] = 'R';
+		trackheader.header.track_id[2] = 'K';
+		trackheader.header.strack = strack;
 
 		FluxmapReader fmr(fluxmap);
 		Bytes fluxdata;
 		ByteWriter fluxdataWriter(fluxdata);
 
 		if (_config.align_with_index())
-			fmr.findEvent(F_BIT_INDEX);
+			fmr.skipToEvent(F_BIT_INDEX);
 
 		int revolution = 0;
 		unsigned revTicks = 0;
@@ -108,12 +107,14 @@ public:
 		while (revolution < 5)
 		{
 			unsigned ticks;
-			uint8_t bits = fmr.getNextEvent(ticks);
+			int event;
+			fmr.getNextEvent(event, ticks);
+
 			ticksSinceLastPulse += ticks;
 			totalTicks += ticks;
 			revTicks += ticks;
 
-			if (fmr.eof() || (bits & F_BIT_INDEX))
+			if (fmr.eof() || (event & F_BIT_INDEX))
 			{
 				auto* revheader = &trackheader.revolution[revolution];
 				write_le32(revheader->offset, startOffset + sizeof(ScpTrack));
@@ -125,7 +126,7 @@ public:
 				startOffset = fluxdataWriter.pos;
 			}
 
-			if (bits & F_BIT_PULSE)
+			if (event & F_BIT_PULSE)
 			{
 				unsigned t = ticksSinceLastPulse * NS_PER_TICK / 25;
 				while (t >= 0x10000)
@@ -138,6 +139,7 @@ public:
 			}
 		}
 
+		_fileheader.revolutions = revolution - 1;
 		write_le32(_fileheader.track[strack], trackdataWriter.pos + sizeof(ScpHeader));
 		trackdataWriter += Bytes((uint8_t*)&trackheader, sizeof(trackheader));
 		trackdataWriter += fluxdata;

lib/fluxsink/vcdfluxsink.cc

@@ -43,7 +43,8 @@ public:
 		while (!fmr.eof())
 		{
 			unsigned ticks;
-			uint8_t bits = fmr.getNextEvent(ticks);
+			int event;
+			fmr.getNextEvent(event, ticks);
 			if (fmr.eof())
 				break;
 
@@ -55,9 +56,9 @@ public:
 				of << fmt::format("#{} ", (uint64_t)(timestamp * NS_PER_TICK));
 			}
 
-			if (bits & F_BIT_PULSE)
+			if (event & F_BIT_PULSE)
 				of << "1p ";
-			if (bits & F_BIT_INDEX)
+			if (event & F_BIT_INDEX)
 				of << "1i ";
 
 			lasttimestamp = timestamp;

lib/fluxsource/cwffluxsource.cc

@@ -139,77 +139,3 @@ std::unique_ptr<FluxSource> FluxSource::createCwfFluxSource(const CwfFluxSourceP
 {
     return std::unique_ptr<FluxSource>(new CwfFluxSource(config));
 }
-
-#if 0
-#include "globals.h"
-#include "fluxmap.h"
-#include "sql.h"
-#include "bytes.h"
-#include "protocol.h"
-
-static std::ifstream inputFile;
-static sqlite3* outputDb;
-static CwfHeader header;
-static double clockRate;
-
-static void syntax()
-{
-    std::cout << "Syntax: fluxengine convert cwftoflux <cwffile> <fluxfile>\n";
-    exit(0);
-}
-
-static void check_for_error()
-{
-    if (inputFile.fail())
-        Error() << fmt::format("I/O error: {}", strerror(errno));
-}
-
-static void read_header()
-{
-}
-
-static void read_track()
-{
-    CwfTrack trackheader;
-    inputFile.read((char*) &trackheader, sizeof(trackheader));
-    check_for_error();
-
-    uint32_t length = Bytes(trackheader.length, 4).reader().read_le32() - sizeof(CwfTrack);
-    unsigned track_number = trackheader.track * header.step;
-    std::cout << fmt::format("{}.{}: {} input bytes; ", track_number, trackheader.side, length)
-        << std::flush;
-
-    std::vector<uint8_t> inputdata(length);
-    inputFile.read((char*) &inputdata[0], length);
-    check_for_error();
-
-
-    std::cout << fmt::format(" {} ms in {} output bytes\n",
-        fluxmap.duration() / 1e6, fluxmap.bytes());
-    sqlWriteFlux(outputDb, track_number, trackheader.side, fluxmap);
-}
-
-int mainConvertCwfToFlux(int argc, const char* argv[])
-{
-    if (argc != 3)
-        syntax();
-    
-    inputFile.open(argv[1], std::ios::in | std::ios::binary);
-    if (!inputFile.is_open())
-		Error() << fmt::format("cannot open input file '{}'", argv[1]);
-
-    outputDb = sqlOpen(argv[2], SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
-    sqlPrepareFlux(outputDb);
-    sqlWriteIntProperty(outputDb, "version", FLUX_VERSION_CURRENT);
-    sqlStmt(outputDb, "BEGIN;");
-
-    read_header();
-    inputFile.seekg(sizeof(header), std::ios::beg);
-    for (unsigned i=0; i<(header.cylinders*header.sides); i++)
-        read_track();
-
-    sqlStmt(outputDb, "COMMIT;");
-    sqlClose(outputDb);
-    return 0;
-}
-#endif

lib/fluxsource/fl2fluxsource.cc

@@ -0,0 +1,62 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "lib/fluxsource/fluxsource.pb.h"
+#include "lib/fl2.pb.h"
+#include "fluxsource/fluxsource.h"
+#include "proto.h"
+#include "fmt/format.h"
+#include <fstream>
+
+class Fl2FluxSource : public FluxSource
+{
+public:
+    Fl2FluxSource(const Fl2FluxSourceProto& config):
+		_config(config)
+    {
+		std::ifstream ifs(_config.filename(), std::ios::in | std::ios::binary);
+		if (!ifs.is_open())
+			Error() << fmt::format("cannot open input file '{}': {}",
+				_config.filename(), strerror(errno));
+
+		if (!_proto.ParseFromIstream(&ifs))
+			Error() << "unable to read input file";
+	}
+
+public:
+    std::unique_ptr<Fluxmap> readFlux(int cylinder, int head)
+    {
+		for (const auto& track : _proto.track())
+		{
+			if ((track.cylinder() == cylinder) && (track.head() == head))
+				return std::make_unique<Fluxmap>(track.flux());
+		}
+
+		return std::unique_ptr<Fluxmap>();
+    }
+
+    void recalibrate() {}
+
+private:
+	void check_for_error(std::ifstream& ifs)
+	{
+		if (ifs.fail())
+			Error() << fmt::format("FL2 read I/O error: {}", strerror(errno));
+	}
+
+private:
+    const Fl2FluxSourceProto& _config;
+	FluxFileProto _proto;
+};
+
+std::unique_ptr<FluxSource> FluxSource::createFl2FluxSource(const Fl2FluxSourceProto& config)
+{
+	char buffer[16];
+	std::ifstream(config.filename(), std::ios::in | std::ios::binary).read(buffer, 16);
+	if (strncmp(buffer, "SQLite format 3", 16) == 0)
+	{
+		std::cerr << "Warning: reading a deprecated flux file format; please upgrade it\n";
+		return FluxSource::createSqliteFluxSource(config.filename());
+	}
+    return std::unique_ptr<FluxSource>(new Fl2FluxSource(config));
+}
+

lib/fluxsource/fluxsource.cc

@@ -39,6 +39,9 @@ std::unique_ptr<FluxSource> FluxSource::create(const FluxSourceProto& config)
 		case FluxSourceProto::kCwf:
 			return createCwfFluxSource(config.cwf());
 
+		case FluxSourceProto::kFl2:
+			return createFl2FluxSource(config.fl2());
+
 		default:
 			Error() << "bad input disk configuration";
 			return std::unique_ptr<FluxSource>();
@@ -47,9 +50,10 @@ std::unique_ptr<FluxSource> FluxSource::create(const FluxSourceProto& config)
 
 void FluxSource::updateConfigForFilename(FluxSourceProto* proto, const std::string& filename)
 {
+
 	static const std::vector<std::pair<std::regex, std::function<void(const std::string&)>>> formats =
 	{
-		{ std::regex("^(.*\\.flux)$"),     [&](const auto& s) { proto->set_fluxfile(s); }},
+		{ std::regex("^(.*\\.flux)$"),     [&](const auto& s) { proto->mutable_fl2()->set_filename(s); }},
 		{ std::regex("^(.*\\.scp)$"),      [&](const auto& s) { proto->mutable_scp()->set_filename(s); }},
 		{ std::regex("^(.*\\.cwf)$"),      [&](const auto& s) { proto->mutable_cwf()->set_filename(s); }},
 		{ std::regex("^erase:$"),          [&](const auto& s) { proto->mutable_erase(); }},

lib/fluxsource/fluxsource.h

@@ -5,6 +5,7 @@
 
 class CwfFluxSourceProto;
 class EraseFluxSourceProto;
+class Fl2FluxSourceProto;
 class FluxSourceProto;
 class FluxSpec;
 class Fluxmap;
@@ -21,6 +22,7 @@ public:
 private:
     static std::unique_ptr<FluxSource> createCwfFluxSource(const CwfFluxSourceProto& config);
     static std::unique_ptr<FluxSource> createEraseFluxSource(const EraseFluxSourceProto& config);
+    static std::unique_ptr<FluxSource> createFl2FluxSource(const Fl2FluxSourceProto& config);
     static std::unique_ptr<FluxSource> createHardwareFluxSource(const HardwareFluxSourceProto& config);
     static std::unique_ptr<FluxSource> createKryofluxFluxSource(const KryofluxFluxSourceProto& config);
     static std::unique_ptr<FluxSource> createScpFluxSource(const ScpFluxSourceProto& config);

lib/fluxsource/fluxsource.proto

@@ -29,10 +29,15 @@ message ScpFluxSourceProto {
 }
 
 message CwfFluxSourceProto {
-	optional string filename = 1 [default = "flux.xwf",
+	optional string filename = 1 [default = "flux.cwf",
 		(help) = ".cwf file to read flux from"];
 }
 
+message Fl2FluxSourceProto {
+	optional string filename = 1 [default = "flux.fl2",
+		(help) = ".fl2 file to read flux from"];
+}
+
 message FluxSourceProto {
 	oneof source {
 		string fluxfile = 1 [default = "name of source flux file"];
@@ -42,6 +47,7 @@ message FluxSourceProto {
 		KryofluxFluxSourceProto kryoflux = 5;
 		ScpFluxSourceProto scp = 6;
 		CwfFluxSourceProto cwf = 7;
+		Fl2FluxSourceProto fl2 = 8;
 	}
 }
 

lib/fluxsource/scpfluxsource.cc

@@ -2,6 +2,7 @@
 #include "fluxmap.h"
 #include "kryoflux.h"
 #include "lib/fluxsource/fluxsource.pb.h"
+#include "lib/utils.h"
 #include "fluxsource/fluxsource.h"
 #include "scp.h"
 #include "proto.h"
@@ -57,11 +58,13 @@ public:
     std::unique_ptr<Fluxmap> readFlux(int track, int side)
     {
 		int strack = strackno(track, side);
+		if (strack >= ARRAY_SIZE(_header.track))
+			return std::unique_ptr<Fluxmap>();
 		uint32_t offset = Bytes(_header.track[strack], 4).reader().read_le32();
 		if (offset == 0)
 			return std::unique_ptr<Fluxmap>();
 
-		ScpTrack trackheader;
+		ScpTrackHeader trackheader;
 		_if.seekg(offset, std::ios::beg);
 		_if.read((char*) &trackheader, sizeof(trackheader));
 		check_for_error();
@@ -71,6 +74,15 @@ public:
 				|| (trackheader.track_id[2] != 'K'))
 			Error() << "corrupt SCP file";
 
+		std::vector<ScpTrackRevolution> revs(_header.revolutions);
+		for (int revolution = 0; revolution < _header.revolutions; revolution++)
+		{
+			ScpTrackRevolution trackrev;
+			_if.read((char*) &trackrev, sizeof(trackrev));
+			check_for_error();
+			revs[revolution] = trackrev;
+		}
+
 		std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
 		nanoseconds_t pending = 0;
 		unsigned inputBytes = 0;
@@ -79,8 +91,8 @@ public:
 			if (revolution != 0)
 				fluxmap->appendIndex();
 
-			uint32_t datalength = Bytes(trackheader.revolution[revolution].length, 4).reader().read_le32();
-			uint32_t dataoffset = Bytes(trackheader.revolution[revolution].offset, 4).reader().read_le32();
+			uint32_t datalength = Bytes(revs[revolution].length, 4).reader().read_le32();
+			uint32_t dataoffset = Bytes(revs[revolution].offset, 4).reader().read_le32();
 
 			Bytes data(datalength*2);
 			_if.seekg(dataoffset + offset, std::ios::beg);

lib/globals.h

@@ -18,6 +18,10 @@
 #define mkdir(A, B) _mkdir(A)
 #endif
 
+template <class T>
+static inline std::vector<T> vector_of(T item)
+{ return std::vector<T> { item }; }
+
 typedef double nanoseconds_t;
 class Bytes;
 
@@ -45,5 +49,4 @@ private:
     std::stringstream _stream;
 };
 
-
 #endif

lib/image.cc

@@ -30,6 +30,9 @@ const std::shared_ptr<Sector>& Image::put(unsigned track, unsigned side, unsigne
 {
 	key_t key = std::make_tuple(track, side, sectorid);
 	std::shared_ptr<Sector> sector = std::make_shared<Sector>();
+	sector->logicalTrack = track;
+	sector->logicalSide = side;
+	sector->logicalSector = sectorid;
 	return _sectors[key] = sector;
 }
 

lib/imagereader/d64imagereader.cc

@@ -16,7 +16,7 @@ public:
 		ImageReader(config)
 	{}
 
-	Image readImage()
+	std::unique_ptr<Image> readImage()
 	{
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
@@ -53,7 +53,7 @@ public:
             return 17;
 		};
 
-        Image image;
+        std::unique_ptr<Image> image(new Image);
         for (int track = 0; track < 40; track++)
         {
 			int numSectors = sectorsPerTrack(track);
@@ -62,7 +62,7 @@ public:
             {
                 for (int sectorId = 0; sectorId < numSectors; sectorId++)
                 {
-					const auto& sector = image.put(track, head, sectorId);
+					const auto& sector = image->put(track, head, sectorId);
                     if ((offset < inputFileSize))
                     {    //still data available sector OK
 						br.seek(offset);
@@ -88,7 +88,7 @@ public:
             }
         }
 
-		image.calculateSize();
+		image->calculateSize();
         return image;
 	}
 };

lib/imagereader/d88imagereader.cc

@@ -0,0 +1,176 @@
+#include "globals.h"
+#include "flags.h"
+#include "sector.h"
+#include "imagereader/imagereader.h"
+#include "image.h"
+#include "proto.h"
+#include "lib/config.pb.h"
+#include "imagereader/imagereaderimpl.h"
+#include "fmt/format.h"
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+
+// reader based on this partial documentation of the D88 format:
+// https://www.pc98.org/project/doc/d88.html
+
+class D88ImageReader : public ImageReader
+{
+public:
+	D88ImageReader(const ImageReaderProto& config):
+		ImageReader(config)
+	{}
+
+	std::unique_ptr<Image> readImage()
+	{
+        std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
+        if (!inputFile.is_open())
+            Error() << "cannot open input file";
+
+        Bytes header(0x24); // read first entry of track table as well
+        inputFile.read((char*) header.begin(), header.size());
+
+        // the DIM header technically has a bit field for sectors present,
+        // however it is currently ignored by this reader
+
+        std::string diskName = header.slice(0, 0x16);
+
+        if (diskName[0])
+            std::cout << "D88: disk name: " << diskName << "\n";
+
+        ByteReader headerReader(header);
+
+        // media flag indicates media density, currently unused
+        char mediaFlag = headerReader.seek(0x1b).read_8();
+
+        inputFile.seekg( 0, std::ios::end );
+        int fileSize = inputFile.tellg();
+
+        int diskSize = headerReader.seek(0x1c).read_le32();
+
+        if (diskSize > fileSize)
+            std::cout << "D88: found multiple disk images. Only using first\n";
+
+        int trackTableEnd = headerReader.seek(0x20).read_le32();
+        int trackTableSize = trackTableEnd - 0x20;
+
+        Bytes trackTable(trackTableSize);
+        inputFile.seekg(0x20);
+        inputFile.read((char*) trackTable.begin(), trackTable.size());
+        ByteReader trackTableReader(trackTable);
+
+        int diskSectorsPerTrack = -1;
+
+        if (config.encoder().format_case() != EncoderProto::FormatCase::FORMAT_NOT_SET)
+            std::cout << "D88: overriding configured format";
+
+        auto ibm = config.mutable_encoder()->mutable_ibm();
+        config.mutable_cylinders()->set_end(0);
+
+        std::unique_ptr<Image> image(new Image);
+        for (int track = 0; track < trackTableSize / 4; track++)
+        {
+            int trackOffset = trackTableReader.seek(track * 4).read_le32();
+            if (trackOffset == 0) continue;
+
+            int currentTrackOffset = trackOffset;
+            int currentTrackCylinder = -1;
+            int currentSectorsInTrack = 0xffff; // don't know # of sectors until we read the first one
+            int trackSectorSize = -1;
+            int trackMfm = -1;
+
+            auto trackdata = ibm->add_trackdata();
+            trackdata->set_clock_rate_khz(500);
+            trackdata->set_track_length_ms(167);
+            auto sectors = trackdata->mutable_sectors();
+
+            for (int sectorInTrack = 0; sectorInTrack < currentSectorsInTrack; sectorInTrack++){
+                Bytes sectorHeader(0x10);
+                inputFile.read((char*) sectorHeader.begin(), sectorHeader.size());
+                ByteReader sectorHeaderReader(sectorHeader);
+                int cylinder = sectorHeaderReader.seek(0).read_8();
+                int head = sectorHeaderReader.seek(1).read_8();
+                int sectorId = sectorHeaderReader.seek(2).read_8();
+                int sectorSize = 128 << sectorHeaderReader.seek(3).read_8();
+                int sectorsInTrack = sectorHeaderReader.seek(4).read_le16();
+                int fm = sectorHeaderReader.seek(6).read_8();
+                int ddam = sectorHeaderReader.seek(7).read_8();
+                int fddStatusCode = sectorHeaderReader.seek(8).read_8();
+                int rpm = sectorHeaderReader.seek(13).read_8();
+                // D88 provides much more sector information that is currently ignored
+                if (ddam != 0)
+                    Error() << "D88: nonzero ddam currently unsupported";
+                if (rpm != 0)
+                    Error() << "D88: 1.44MB 300rpm formats currently unsupported";
+                if (fddStatusCode != 0)
+                    Error() << "D88: nonzero fdd status codes are currently unsupported";
+                if (currentSectorsInTrack == 0xffff) {
+                    currentSectorsInTrack = sectorsInTrack;
+                } else if (currentSectorsInTrack != sectorsInTrack) {
+                    Error() << "D88: mismatched number of sectors in track";
+                }
+                if (diskSectorsPerTrack < 0) {
+                    diskSectorsPerTrack = sectorsInTrack;
+                } else if (diskSectorsPerTrack != sectorsInTrack) {
+                    Error() << "D88: varying numbers of sectors per track is currently unsupported";
+                }
+                if (currentTrackCylinder < 0) {
+                    currentTrackCylinder = cylinder;
+                } else if (currentTrackCylinder != cylinder) {
+                    Error() << "D88: all sectors in a track must belong to the same cylinder";
+                }
+                if (trackSectorSize < 0) {
+                    trackSectorSize = sectorSize;
+                    // this is the first sector we've read, use it settings for per-track data
+                    trackdata->set_cylinder(cylinder);
+                    trackdata->set_head(head);
+                    trackdata->set_sector_size(sectorSize);
+                    trackdata->set_use_fm(fm);
+                    if (fm) {
+                        trackdata->set_gap_fill_byte(0xffff);
+                        trackdata->set_idam_byte(0xf57e);
+                        trackdata->set_dam_byte(0xf56f);
+                    }
+                    // create timings to approximately match N88-BASIC
+                    if (sectorSize <= 128) {
+                        trackdata->set_gap0(0x1b);
+                        trackdata->set_gap2(0x09);
+                        trackdata->set_gap3(0x1b);
+                    } else if (sectorSize <= 256) {
+                        trackdata->set_gap0(0x36);
+                        trackdata->set_gap3(0x36);
+                    }
+                } else if (trackSectorSize != sectorSize) {
+                    Error() << "D88: multiple sector sizes per track are currently unsupported";
+                }
+                Bytes data(sectorSize);
+                inputFile.read((char*) data.begin(), data.size());
+                const auto& sector = image->put(cylinder, head, sectorId);
+                sector->status = Sector::OK;
+                sector->logicalTrack = cylinder;
+                sector->physicalCylinder = cylinder;
+                sector->logicalSide = sector->physicalHead = head;
+                sector->logicalSector = sectorId;
+                sector->data = data;
+
+                sectors->add_sector(sectorId);
+                if (config.cylinders().end() < cylinder)
+                    config.mutable_cylinders()->set_end(cylinder);
+            }
+        }
+
+        image->calculateSize();
+        const Geometry& geometry = image->getGeometry();
+        std::cout << fmt::format("D88: read {} tracks, {} sides\n",
+                        geometry.numTracks, geometry.numSides);
+        return image;
+    }
+
+};
+
+std::unique_ptr<ImageReader> ImageReader::createD88ImageReader(
+	const ImageReaderProto& config)
+{
+    return std::unique_ptr<ImageReader>(new D88ImageReader(config));
+}
+

lib/imagereader/dimimagereader.cc

@@ -0,0 +1,150 @@
+#include "globals.h"
+#include "flags.h"
+#include "sector.h"
+#include "imagereader/imagereader.h"
+#include "image.h"
+#include "proto.h"
+#include "lib/config.pb.h"
+#include "imagereader/imagereaderimpl.h"
+#include "fmt/format.h"
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+
+// reader based on this partial documentation of the DIM format:
+// https://www.pc98.org/project/doc/dim.html
+
+class DimImageReader : public ImageReader
+{
+public:
+	DimImageReader(const ImageReaderProto& config):
+		ImageReader(config)
+	{}
+
+	std::unique_ptr<Image> readImage()
+	{
+        std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
+        if (!inputFile.is_open())
+            Error() << "cannot open input file";
+
+        Bytes header(256);
+        inputFile.read((char*) header.begin(), header.size());
+        if (header.slice(0xAB, 13) != Bytes("DIFC HEADER  "))
+            Error() << "DIM: could not find DIM header, is this a DIM file?";
+
+        // the DIM header technically has a bit field for sectors present,
+        // however it is currently ignored by this reader
+
+        char mediaByte = header[0];
+        int tracks;
+        int sectorsPerTrack;
+        int sectorSize;
+        switch (mediaByte) {
+            case 0:
+                tracks = 77;
+                sectorsPerTrack = 8;
+                sectorSize = 1024;
+                break;
+            case 1:
+                tracks = 80;
+                sectorsPerTrack = 9;
+                sectorSize = 1024;
+                break;
+            case 2:
+                tracks = 80;
+                sectorsPerTrack = 15;
+                sectorSize = 512;
+                break;
+            case 3:
+                tracks = 80;
+                sectorsPerTrack = 18;
+                sectorSize = 512;
+                break;
+            default:
+                Error() << "DIM: unsupported media byte";
+                break;
+        }
+
+        std::unique_ptr<Image> image(new Image);
+		int trackCount = 0;
+        for (int track = 0; track < tracks; track++)
+        {
+			if (inputFile.eof())
+				break;
+			int physicalCylinder = track;
+
+            for (int side = 0; side < 2; side++)
+            {
+                std::vector<unsigned> sectors;
+                for (int i = 0; i < sectorsPerTrack; i++)
+                    sectors.push_back(i + 1);
+
+                for (int sectorId : sectors)
+                {
+                    Bytes data(sectorSize);
+                    inputFile.read((char*) data.begin(), data.size());
+
+					const auto& sector = image->put(physicalCylinder, side, sectorId);
+                    sector->status = Sector::OK;
+                    sector->logicalTrack = track;
+					sector->physicalCylinder = physicalCylinder;
+                    sector->logicalSide = sector->physicalHead = side;
+                    sector->logicalSector = sectorId;
+                    sector->data = data;
+                }
+            }
+
+			trackCount++;
+        }
+
+        if (config.encoder().format_case() == EncoderProto::FormatCase::FORMAT_NOT_SET)
+        {
+            auto ibm = config.mutable_encoder()->mutable_ibm();
+            auto trackdata = ibm->add_trackdata();
+            trackdata->set_clock_rate_khz(500);
+            auto sectors = trackdata->mutable_sectors();
+            switch (mediaByte) {
+                case 0x00:
+                    std::cout << "DIM: automatically setting format to 1.2MB (1024 byte sectors)\n";
+                    config.mutable_cylinders()->set_end(76);
+                    trackdata->set_track_length_ms(167);
+                    trackdata->set_sector_size(1024);
+                    for (int i = 0; i < 9; i++)
+                        sectors->add_sector(i);
+                    break;
+                case 0x02:
+                    std::cout << "DIM: automatically setting format to 1.2MB (512 byte sectors)\n";
+                    trackdata->set_track_length_ms(167);
+                    trackdata->set_sector_size(512);
+                    for (int i = 0; i < 15; i++)
+                        sectors->add_sector(i);
+                    break;
+                case 0x03:
+                    std::cout << "DIM: automatically setting format to 1.44MB\n";
+                    trackdata->set_track_length_ms(200);
+                    trackdata->set_sector_size(512);
+                    for (int i = 0; i < 18; i++)
+                        sectors->add_sector(i);
+                    break;
+                default:
+                    Error() << fmt::format("DIM: unknown media byte 0x%02x, could not determine write profile automatically", mediaByte);
+                    break;
+            }
+        }
+
+		image->calculateSize();
+		const Geometry& geometry = image->getGeometry();
+        std::cout << fmt::format("DIM: read {} tracks, {} sides, {} kB total\n",
+                        geometry.numTracks, geometry.numSides,
+						((int)inputFile.tellg() - 256) / 1024);
+        return image;
+	}
+
+};
+
+std::unique_ptr<ImageReader> ImageReader::createDimImageReader(
+	const ImageReaderProto& config)
+{
+    return std::unique_ptr<ImageReader>(new DimImageReader(config));
+}
+

lib/imagereader/diskcopyimagereader.cc

@@ -16,7 +16,7 @@ public:
 		ImageReader(config)
 	{}
 
-	Image readImage()
+	std::unique_ptr<Image> readImage()
 	{
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
@@ -89,7 +89,7 @@ public:
 		uint32_t dataPtr = 0x54;
 		uint32_t tagPtr = dataPtr + dataSize;
 
-        Image image;
+        std::unique_ptr<Image> image(new Image);
         for (int track = 0; track < numCylinders; track++)
         {
 			int numSectors = sectorsPerTrack(track);
@@ -105,7 +105,7 @@ public:
 					Bytes tag = br.read(12);
 					tagPtr += 12;
 
-                    const auto& sector = image.put(track, head, sectorId);
+                    const auto& sector = image->put(track, head, sectorId);
                     sector->status = Sector::OK;
                     sector->logicalTrack = sector->physicalCylinder = track;
                     sector->logicalSide = sector->physicalHead = head;
@@ -115,13 +115,13 @@ public:
             }
         }
 
-		Geometry g;
-		g.numTracks = numCylinders;
-		g.numSides = numHeads;
-		g.numSectors = 12;
-		g.sectorSize = 512 + 12;
-		g.irregular = true;
-		image.setGeometry(g);
+		image->setGeometry({
+			.numTracks = numCylinders,
+			.numSides = numHeads,
+			.numSectors = 12,
+			.sectorSize = 512 + 12,
+			.irregular = true
+		});
         return image;
 	}
 };

lib/imagereader/fdiimagereader.cc

@@ -0,0 +1,94 @@
+#include "globals.h"
+#include "flags.h"
+#include "sector.h"
+#include "imagereader/imagereader.h"
+#include "image.h"
+#include "lib/config.pb.h"
+#include "imagereader/imagereaderimpl.h"
+#include "fmt/format.h"
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+
+// reader based on this partial documentation of the FDI format:
+// https://www.pc98.org/project/doc/hdi.html
+
+class FdiImageReader : public ImageReader
+{
+public:
+	FdiImageReader(const ImageReaderProto& config):
+		ImageReader(config)
+	{}
+
+	std::unique_ptr<Image> readImage()
+	{
+        std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
+        if (!inputFile.is_open())
+            Error() << "cannot open input file";
+
+        Bytes header(32);
+        inputFile.read((char*) header.begin(), header.size());
+        ByteReader headerReader(header);
+        if (headerReader.seek(0).read_le32() != 0)
+            Error() << "FDI: could not find FDI header, is this a FDI file?";
+
+        // we currently don't use fddType but it could be used to automatically select
+        // profile parameters in the future
+        //
+        int fddType = headerReader.seek(4).read_le32();
+        int headerSize = headerReader.seek(0x08).read_le32();
+        int sectorSize = headerReader.seek(0x10).read_le32();
+        int sectorsPerTrack = headerReader.seek(0x14).read_le32();
+        int sides = headerReader.seek(0x18).read_le32();
+        int tracks = headerReader.seek(0x1c).read_le32();
+
+        inputFile.seekg(headerSize);
+
+        std::unique_ptr<Image> image(new Image);
+		int trackCount = 0;
+        for (int track = 0; track < tracks; track++)
+        {
+			if (inputFile.eof())
+				break;
+			int physicalCylinder = track;
+
+            for (int side = 0; side < sides; side++)
+            {
+                std::vector<unsigned> sectors;
+                for (int i = 0; i < sectorsPerTrack; i++)
+                    sectors.push_back(i + 1);
+
+                for (int sectorId : sectors)
+                {
+                    Bytes data(sectorSize);
+                    inputFile.read((char*) data.begin(), data.size());
+
+					const auto& sector = image->put(physicalCylinder, side, sectorId);
+                    sector->status = Sector::OK;
+                    sector->logicalTrack = track;
+					sector->physicalCylinder = physicalCylinder;
+                    sector->logicalSide = sector->physicalHead = side;
+                    sector->logicalSector = sectorId;
+                    sector->data = data;
+                }
+            }
+
+			trackCount++;
+        }
+
+		image->calculateSize();
+		const Geometry& geometry = image->getGeometry();
+        std::cout << fmt::format("FDI: read {} tracks, {} sides, {} kB total\n",
+                        geometry.numTracks, geometry.numSides,
+						((int)inputFile.tellg() - headerSize) / 1024);
+        return image;
+	}
+
+};
+
+std::unique_ptr<ImageReader> ImageReader::createFdiImageReader(
+	const ImageReaderProto& config)
+{
+    return std::unique_ptr<ImageReader>(new FdiImageReader(config));
+}
+

lib/imagereader/imagereader.cc

@@ -14,6 +14,15 @@ std::unique_ptr<ImageReader> ImageReader::create(const ImageReaderProto& config)
 {
 	switch (config.format_case())
 	{
+		case ImageReaderProto::kDim:
+			return ImageReader::createDimImageReader(config);
+
+		case ImageReaderProto::kD88:
+			return ImageReader::createD88ImageReader(config);
+
+		case ImageReaderProto::kFdi:
+			return ImageReader::createFdiImageReader(config);
+
 		case ImageReaderProto::kImd:
 			return ImageReader::createIMDImageReader(config);
 
@@ -32,6 +41,9 @@ std::unique_ptr<ImageReader> ImageReader::create(const ImageReaderProto& config)
 		case ImageReaderProto::kNsi:
 			return ImageReader::createNsiImageReader(config);
 
+		case ImageReaderProto::kTd0:
+			return ImageReader::createTd0ImageReader(config);
+
 		default:
 			Error() << "bad input file config";
 			return std::unique_ptr<ImageReader>();
@@ -46,10 +58,16 @@ void ImageReader::updateConfigForFilename(ImageReaderProto* proto, const std::st
 		{".jv3",      [&]() { proto->mutable_jv3(); }},
 		{".d64",      [&]() { proto->mutable_d64(); }},
 		{".d81",      [&]() { proto->mutable_img(); }},
+		{".d88",      [&]() { proto->mutable_d88(); }},
+		{".dim",      [&]() { proto->mutable_dim(); }},
 		{".diskcopy", [&]() { proto->mutable_diskcopy(); }},
+		{".fdi",      [&]() { proto->mutable_fdi(); }},
+		{".imd",      [&]() { proto->mutable_imd(); }},
 		{".img",      [&]() { proto->mutable_img(); }},
 		{".st",       [&]() { proto->mutable_img(); }},
 		{".nsi",      [&]() { proto->mutable_nsi(); }},
+		{".td0",      [&]() { proto->mutable_td0(); }},
+		{".xdf",      [&]() { proto->mutable_img(); }},
 	};
 
 	for (const auto& it : formats)
@@ -85,4 +103,3 @@ void getTrackFormat(const ImgInputOutputProto& config,
 		trackdata.MergeFrom(f);
 	}
 }
-

lib/imagereader/imagereader.h

@@ -1,9 +1,10 @@
 #ifndef IMAGEREADER_H
 #define IMAGEREADER_H
 
+#include "image.h"
+
 class ImageSpec;
 class ImageReaderProto;
-class Image;
 
 class ImageReader
 {
@@ -22,9 +23,13 @@ public:
     static std::unique_ptr<ImageReader> createJv3ImageReader(const ImageReaderProto& config);
     static std::unique_ptr<ImageReader> createIMDImageReader(const ImageReaderProto& config);
     static std::unique_ptr<ImageReader> createNsiImageReader(const ImageReaderProto& config);
+    static std::unique_ptr<ImageReader> createTd0ImageReader(const ImageReaderProto& config);
+    static std::unique_ptr<ImageReader> createDimImageReader(const ImageReaderProto& config);
+    static std::unique_ptr<ImageReader> createFdiImageReader(const ImageReaderProto& config);
+    static std::unique_ptr<ImageReader> createD88ImageReader(const ImageReaderProto& config);
 
 public:
-	virtual Image readImage() = 0;
+	virtual std::unique_ptr<Image> readImage() = 0;
 
 protected:
 	const ImageReaderProto& _config;

lib/imagereader/imagereader.proto

@@ -37,6 +37,10 @@ message ImdInputProto {}
 message Jv3InputProto {}
 message D64InputProto {}
 message NsiInputProto {}
+message Td0InputProto {}
+message DimInputProto {}
+message FdiInputProto {}
+message D88InputProto {}
 
 message ImageReaderProto {
 	optional string filename = 1 [(help) = "filename of input sector image"];
@@ -47,6 +51,10 @@ message ImageReaderProto {
 		Jv3InputProto jv3 = 5;
 		D64InputProto d64 = 6;
 		NsiInputProto nsi = 7;
+		Td0InputProto td0 = 8;
+		DimInputProto dim = 9;
+		FdiInputProto fdi = 10;
+		D88InputProto d88 = 11;
 	}
 }
 

lib/imagereader/imdimagereader.cc

@@ -3,6 +3,7 @@
 #include "sector.h"
 #include "imagereader/imagereader.h"
 #include "image.h"
+#include "proto.h"
 #include "lib/config.pb.h"
 #include "fmt/format.h"
 #include <algorithm>
@@ -11,68 +12,68 @@
 
 static unsigned getModulationandSpeed(uint8_t flags, bool *mfm)
 {
-	switch (flags)
-		{
-			case 0: /* 500 kbps FM */
-				//clockRateKhz.setDefaultValue(250);
-				*mfm = false;
-				return 500;
-				break;
+    switch (flags)
+        {
+            case 0: /* 500 kbps FM */
+                //clockRateKhz.setDefaultValue(250);
+                *mfm = false;
+                return 500;
+                break;
 
-			case 1: /* 300 kbps FM */
-				*mfm  = false;
-				return 300;
-				
-				break;
+            case 1: /* 300 kbps FM */
+                *mfm  = false;
+                return 300;
+                
+                break;
 
-			case 2: /* 250 kbps FM */
-				*mfm  = false;
-				return 250;
-				break;
+            case 2: /* 250 kbps FM */
+                *mfm  = false;
+                return 250;
+                break;
 
-			case 3: /* 500 kbps MFM */
-				*mfm  = true;
-				return 500;
-				break;
+            case 3: /* 500 kbps MFM */
+                *mfm  = true;
+                return 500;
+                break;
 
-			case 4: /* 300 kbps MFM */
-				*mfm  = true;
-				return 300;
-				break;
+            case 4: /* 300 kbps MFM */
+                *mfm  = true;
+                return 300;
+                break;
 
-			case 5: /* 250 kbps MFM */
-				*mfm  = true;
-				return 250;
-				break;
+            case 5: /* 250 kbps MFM */
+                *mfm  = true;
+                return 250;
+                break;
 
-			default:
-				Error() << fmt::format("don't understand IMD disks with this modulation and speed {}", flags);
-				throw 0;
-		}
+            default:
+                Error() << fmt::format("don't understand IMD disks with this modulation and speed {}", flags);
+                throw 0;
+        }
 }
 
 struct TrackHeader
 {
-	uint8_t ModeValue;
-	uint8_t track;
-	uint8_t Head;
-	uint8_t numSectors;
-	uint8_t SectorSize;
+    uint8_t ModeValue;
+    uint8_t track;
+    uint8_t Head;
+    uint8_t numSectors;
+    uint8_t SectorSize;
 };
 
 static unsigned getSectorSize(uint8_t flags)
 {
-	switch (flags)
-	{
-		case 0: return 128;
-		case 1: return 256;
-		case 2: return 512;
-		case 3: return 1024;
-		case 4: return 2048;
-		case 5: return 4096;
-		case 6: return 8192;
-	}
-	Error() << "not reachable";
+    switch (flags)
+    {
+        case 0: return 128;
+        case 1: return 256;
+        case 2: return 512;
+        case 3: return 1024;
+        case 4: return 2048;
+        case 5: return 4096;
+        case 6: return 8192;
+    }
+    Error() << "not reachable";
 }
 
 
@@ -85,204 +86,171 @@ static unsigned getSectorSize(uint8_t flags)
 class IMDImageReader : public ImageReader
 {
 public:
-	IMDImageReader(const ImageReaderProto& config):
-		ImageReader(config)
-	{}
+    IMDImageReader(const ImageReaderProto& config):
+        ImageReader(config)
+    {}
 
-	Image readImage()
-	/*
-	IMAGE FILE FORMAT
-	The overall layout of an ImageDisk .IMD image file is:
-	IMD v.vv: dd/mm/yyyy hh:mm:ss
-	Comment (ASCII only - unlimited size)
-	1A byte - ASCII EOF character
-	- For each track on the disk:
-	1 byte Mode value							see getModulationspeed for definition		
-	1 byte Cylinder
-	1 byte Head
-	1 byte number of sectors in track			
-	1 byte sector size							see getsectorsize for definition
-	sector numbering map
-	sector cylinder map (optional)				definied in high byte of head (since head is 0 or 1)
-	sector head map (optional)					definied in high byte of head (since head is 0 or 1)
-	sector data records
-	<End of file>
-	*/
-	{
-		//Read File
+    std::unique_ptr<Image> readImage()
+    /*
+    IMAGE FILE FORMAT
+    The overall layout of an ImageDisk .IMD image file is:
+    IMD v.vv: dd/mm/yyyy hh:mm:ss
+    Comment (ASCII only - unlimited size)
+    1A byte - ASCII EOF character
+    - For each track on the disk:
+    1 byte Mode value                           see getModulationspeed for definition       
+    1 byte Cylinder
+    1 byte Head
+    1 byte number of sectors in track           
+    1 byte sector size                          see getsectorsize for definition
+    sector numbering map
+    sector cylinder map (optional)              definied in high byte of head (since head is 0 or 1)
+    sector head map (optional)                  definied in high byte of head (since head is 0 or 1)
+    sector data records
+    <End of file>
+    */
+    {
+        //Read File
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
             Error() << "cannot open input file";
-		//define some variables
-		bool mfm = false;	//define coding just to show in comment for setting the right write parameters
-		inputFile.seekg(0, inputFile.end);
-		int inputFileSize = inputFile.tellg();	// determine filesize
-		inputFile.seekg(0, inputFile.beg);
-		Bytes data;
-		data.writer() += inputFile;
-		ByteReader br(data);
-		Image image;
-		TrackHeader header = {0, 0, 0, 0, 0};
+        //define some variables
+        bool mfm = false;   //define coding just to show in comment for setting the right write parameters
+        inputFile.seekg(0, inputFile.end);
+        int inputFileSize = inputFile.tellg();  // determine filesize
+        inputFile.seekg(0, inputFile.beg);
+        Bytes data;
+        data.writer() += inputFile;
+        ByteReader br(data);
+        std::unique_ptr<Image> image(new Image);
+        TrackHeader header = {0, 0, 0, 0, 0};
 
-		unsigned n = 0;
-		unsigned headerPtr = 0;
-		unsigned Modulation_Speed = 0;
-		unsigned sectorSize = 0;
-		std::string sector_skew;
-		int b; 	
-		unsigned char comment[8192]; //i choose a fixed value. dont know how to make dynamic arrays in C++. This should be enough
-		// Read comment
-		while ((b = br.read_8()) != EOF && b != 0x1A)
-		{
-			comment[n++] = (unsigned char)b;
-		}
-		headerPtr = n; //set pointer to after comment
-		comment[n] = '\0'; // null-terminate the string
-		//write comment to screen
-		std::cout 	<< "Comment in IMD image:\n"
-					<< fmt::format("{}\n",
-					comment);
+        unsigned n = 0;
+        unsigned headerPtr = 0;
+        unsigned Modulation_Speed = 0;
+        unsigned sectorSize = 0;
+        std::string sector_skew;
+        int b;  
+        unsigned char comment[8192]; //i choose a fixed value. dont know how to make dynamic arrays in C++. This should be enough
+        // Read comment
+        while ((b = br.read_8()) != EOF && b != 0x1A)
+        {
+            comment[n++] = (unsigned char)b;
+        }
+        headerPtr = n; //set pointer to after comment
+        comment[n] = '\0'; // null-terminate the string
+        //write comment to screen
+        std::cout   << "Comment in IMD image:\n"
+                    << fmt::format("{}\n",
+                    comment);
 
-		//first read header
-		for (;;)
-		{
-			if (headerPtr >= inputFileSize-1)
-			{
-				break;
-			}
-			header.ModeValue = br.read_8();
-			headerPtr++;
-			Modulation_Speed = getModulationandSpeed(header.ModeValue, &mfm);
-			header.track = br.read_8();
-			headerPtr++;
-			header.Head = br.read_8();
-			headerPtr++;
-			header.numSectors = br.read_8();
-			headerPtr++;
-			header.SectorSize = br.read_8();
-			headerPtr++;
-			sectorSize = getSectorSize(header.SectorSize);
+        //first read header
+        for (;;)
+        {
+            if (headerPtr >= inputFileSize-1)
+            {
+                break;
+            }
+            header.ModeValue = br.read_8();
+            headerPtr++;
+            Modulation_Speed = getModulationandSpeed(header.ModeValue, &mfm);
+            header.track = br.read_8();
+            headerPtr++;
+            header.Head = br.read_8();
+            headerPtr++;
+            header.numSectors = br.read_8();
+            headerPtr++;
+            header.SectorSize = br.read_8();
+            headerPtr++;
+            sectorSize = getSectorSize(header.SectorSize);
 
-			//Read optional cylinder map To Do
+            //Read optional cylinder map To Do
 
-			//Read optional sector head map To Do
+            //Read optional sector head map To Do
 
-			//read sector numbering map
-			std::vector<unsigned> sector_map(header.numSectors);
-			bool blnBaseOne = false; 
-			sector_skew.clear();
-			for (b = 0;  b < header.numSectors; b++)
-			{	
-				sector_map[b] = br.read_8();
-				sector_skew.push_back(sector_map[b] + '0');
-				if (b == 0) //first sector see if base is 0 or 1 Fluxengine wants 0
-					{
-					if (sector_map[b]==1)
-						{
-							blnBaseOne = true;
-						}
-					}
-				if (blnBaseOne==true)
-				{
-					sector_map[b] = (sector_map[b]-1);
-				}
-				headerPtr++;
-			}
-			//read the sectors
-			for (int s = 0; s < header.numSectors; s++)
-			{
-				Bytes sectordata;
-				const auto& sector = image.put(header.track, header.Head, sector_map[s]);
-				//read the status of the sector
-				unsigned int Status_Sector = br.read_8();
-				headerPtr++;
+            //read sector numbering map
+            std::vector<unsigned> sector_map(header.numSectors);
+            sector_skew.clear();
+            for (int b = 0;  b < header.numSectors; b++)
+            {   
+                sector_map[b] = br.read_8();
+                headerPtr++;
+            }
 
-				switch (Status_Sector)
-				{
-					case 0: /* Sector data unavailable - could not be read */
+            //read the sectors
+            for (int s = 0; s < header.numSectors; s++)
+            {
+                Bytes sectordata;
+                const auto& sector = image->put(header.track, header.Head, sector_map[s]);
+                //read the status of the sector
+                unsigned int Status_Sector = br.read_8();
+                headerPtr++;
 
-						break;
+                switch (Status_Sector)
+                {
+                    case 0: /* Sector data unavailable - could not be read */
+                        break;
 
-					case 1: /* Normal data: (Sector Size) bytes follow */
-						sectordata = br.read(sectorSize);
-						headerPtr += sectorSize;
-						sector->data.writer().append(sectordata);
+                    case 1: /* Normal data: (Sector Size) bytes follow */
+                        sectordata = br.read(sectorSize);
+                        headerPtr += sectorSize;
+                        sector->data.writer().append(sectordata);
+                        break;
 
-						break;
+                    case 2: /* Compressed: All bytes in sector have same value (xx) */
+                        sectordata = br.read(1);
+                        headerPtr++;
+                        sector->data.writer().append(sectordata);
 
-					case 2: /* Compressed: All bytes in sector have same value (xx) */
-						sectordata = br.read(1);
-						headerPtr++;
-						sector->data.writer().append(sectordata);
+                        for (int k = 1; k < sectorSize; k++)
+                        {
+                            //fill data till sector is full
+                            sector->data.writer().append(sectordata);
+                        }
+                        break;
 
-						for (int k = 1; k < sectorSize; k++)
-						{
-							//fill data till sector is full
-							sector->data.writer().append(sectordata);
-						}
+                    case 3: /* Normal data with "Deleted-Data address mark" */
+                    case 4: /* Compressed with "Deleted-Data address mark"*/
+                    case 5: /* Normal data read with data error */
+                    case 6: /* Compressed read with data error" */
+                    case 7: /* Deleted data read with data error" */
+                    case 8: /* Compressed, Deleted read with data error" */
+                    default:
+                        Error() << fmt::format("don't understand IMD disks with sector status {}", Status_Sector);
+                }       
+                sector->status = Sector::OK;
+                sector->logicalTrack = sector->physicalCylinder = header.track;
+                sector->logicalSide = sector->physicalHead = header.Head;
+                sector->logicalSector = (sector_map[s]);
+            }
 
-						break;
+        }
+        //Write format detected in IMD image to screen to help user set the right write parameters
 
-					case 3: /* Normal data with "Deleted-Data address mark" */
+        image->setGeometry({
+            .numTracks = header.track,
+            .numSides = header.Head + 1U,
+            .numSectors = header.numSectors,
+            .sectorSize = sectorSize
+        });
 
-						break;
-
-					case 4: /* Compressed with "Deleted-Data address mark"*/
-					
-						break;
-
-					case 5: /* Normal data read with data error */
-					
-						break;
-
-					case 6: /* Compressed read with data error" */
-
-						break;
-
-					case 7: /* Deleted data read with data error" */
-					
-						break;
-
-					case 8: /* Compressed, Deleted read with data error" */
-					
-						break;
-
-					default:
-						Error() << fmt::format("don't understand IMD disks with sector status {}", Status_Sector);
-				}		
-				sector->status = Sector::OK;
-				sector->logicalTrack = sector->physicalCylinder = header.track;
-				sector->logicalSide = sector->physicalHead = header.Head;
-				sector->logicalSector = (sector_map[s]);
-			}
-
-  		}
-		//Write format detected in IMD image to screen to help user set the right write parameters
-
-		Geometry g;
-		g.numTracks = header.track;
-		g.numSides = header.Head + 1U;
-		g.numSectors = header.numSectors;
-		g.sectorSize = sectorSize;
-		image.setGeometry(g);
-
-		size_t headSize = header.numSectors * sectorSize;
+        size_t headSize = header.numSectors * sectorSize;
         size_t trackSize = headSize * (header.Head + 1);
 
-		std::cout 	<< "reading IMD image\n"
-					<< fmt::format("{} tracks, {} heads; {}; {} kbps; {} sectoren; sectorsize {}; sectormap {}; {} kB total \n",
-					header.track, header.Head + 1,
-					mfm ? "MFM" : "FM",
-					Modulation_Speed, header.numSectors, sectorSize, sector_skew, (header.track+1) * trackSize / 1024);
+        std::cout << fmt::format("IMD: {} tracks, {} heads; {}; {} kbps; {} sectors; sectorsize {};\n"
+                                 "     sectormap {}; {} kB total\n",
+                    header.track, header.Head + 1,
+                    mfm ? "MFM" : "FM",
+                    Modulation_Speed, header.numSectors, sectorSize, sector_skew, (header.track+1) * trackSize / 1024);
 
         return image;
- 	}
+    }
 };
 
 std::unique_ptr<ImageReader> ImageReader::createIMDImageReader(
-	const ImageReaderProto& config)
+    const ImageReaderProto& config)
 {
     return std::unique_ptr<ImageReader>(new IMDImageReader(config));
 }
 
-
+// vim: ts=4 sw=4 et

lib/imagereader/imgimagereader.cc

@@ -17,7 +17,7 @@ public:
 		ImageReader(config)
 	{}
 
-	Image readImage()
+	std::unique_ptr<Image> readImage()
 	{
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
@@ -26,7 +26,7 @@ public:
 		if (!_config.img().tracks() || !_config.img().sides())
 			Error() << "IMG: bad configuration; did you remember to set the tracks, sides and trackdata fields?";
 
-        Image image;
+        std::unique_ptr<Image> image(new Image);
 		int trackCount = 0;
         for (int track = 0; track < _config.img().tracks(); track++)
         {
@@ -44,7 +44,7 @@ public:
                     Bytes data(trackdata.sector_size());
                     inputFile.read((char*) data.begin(), data.size());
 
-					const auto& sector = image.put(physicalCylinder, side, sectorId);
+					const auto& sector = image->put(physicalCylinder, side, sectorId);
                     sector->status = Sector::OK;
                     sector->logicalTrack = track;
 					sector->physicalCylinder = physicalCylinder;
@@ -57,8 +57,8 @@ public:
 			trackCount++;
         }
 
-		image.calculateSize();
-		const Geometry& geometry = image.getGeometry();
+		image->calculateSize();
+		const Geometry& geometry = image->getGeometry();
         std::cout << fmt::format("IMG: read {} tracks, {} sides, {} kB total\n",
                         geometry.numTracks, geometry.numSides,
 						inputFile.tellg() / 1024);

lib/imagereader/jv3imagereader.cc

@@ -81,7 +81,7 @@ public:
 		ImageReader(config)
 	{}
 
-	Image readImage()
+	std::unique_ptr<Image> readImage()
 	{
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
@@ -90,7 +90,7 @@ public:
 		inputFile.seekg( 0, std::ios::end);
 		unsigned inputFileSize = inputFile.tellg();
 		unsigned headerPtr = 0;
-		Image image;
+        std::unique_ptr<Image> image(new Image);
 		for (;;)
 		{
 			unsigned dataPtr = headerPtr + 2901*3 + 1;
@@ -110,7 +110,7 @@ public:
 					inputFile.read((char*) data.begin(), sectorSize);
 
 					unsigned head = !!(header.flags & JV3_SIDE);
-					const auto& sector = image.put(header.track, head, header.sector);
+					const auto& sector = image->put(header.track, head, header.sector);
                     sector->status = Sector::OK;
                     sector->logicalTrack = sector->physicalCylinder = header.track;
                     sector->logicalSide = sector->physicalHead = head;
@@ -127,7 +127,7 @@ public:
 			headerPtr = dataPtr;
 		}
 
-		image.calculateSize();
+		image->calculateSize();
         return image;
 	}
 };

lib/imagereader/nsiimagereader.cc

@@ -18,7 +18,7 @@ public:
 		ImageReader(config)
 	{}
 
-	Image readImage()
+	std::unique_ptr<Image> readImage()
 	{
         std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
         if (!inputFile.is_open())
@@ -64,7 +64,7 @@ public:
                         numCylinders * numHeads * trackSize / 1024)
                 << std::endl;
 
-        Image image;
+        std::unique_ptr<Image> image(new Image);
         unsigned sectorFileOffset;
 
         for (unsigned head = 0; head < numHeads; head++)
@@ -87,7 +87,7 @@ public:
                     Bytes data(sectorSize);
                     inputFile.read((char*) data.begin(), sectorSize);
 
-                    const auto& sector = image.put(track, head, sectorId);
+                    const auto& sector = image->put(track, head, sectorId);
                     sector->status = Sector::OK;
                     sector->logicalTrack = sector->physicalCylinder = track;
                     sector->logicalSide = sector->physicalHead = head;
@@ -97,12 +97,12 @@ public:
             }
         }
 
-		Geometry g;
-		g.numTracks = numCylinders;
-		g.numSides = numHeads;
-		g.numSectors = numSectors;
-		g.sectorSize = sectorSize;
-		image.setGeometry(g);
+		image->setGeometry({
+			.numTracks = numCylinders,
+			.numSides = numHeads,
+			.numSectors = numSectors,
+			.sectorSize = sectorSize
+		});
         return image;
 	}
 };

lib/imagereader/td0imagereader.cc

@@ -0,0 +1,205 @@
+#include "globals.h"
+#include "flags.h"
+#include "sector.h"
+#include "imagereader/imagereader.h"
+#include "image.h"
+#include "crc.h"
+#include "fmt/format.h"
+#include "lib/config.pb.h"
+#include "fmt/format.h"
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+
+/* The best description of the Teledisk format I've found is available here:
+ *
+ * https://web.archive.org/web/20210420230238/http://dunfield.classiccmp.org/img47321/td0notes.txt
+ *
+ * Header:
+ *
+ * Signature                   (2 bytes); TD for normal, td for compressed
+ * Sequence                    (1 byte)
+ * Checksequence               (1 byte)
+ * Teledisk version            (1 byte)
+ * Data rate                   (1 byte)
+ * Drive type                  (1 byte)
+ * Stepping                    (1 byte)
+ * DOS allocation flag         (1 byte)
+ * Sides                       (1 byte)
+ * Cyclic Redundancy Check     (2 bytes)
+ */
+
+enum
+{
+	TD0_ENCODING_RAW = 0,
+	TD0_ENCODING_REPEATED = 1,
+	TD0_ENCODING_RLE = 2,
+
+	TD0_FLAG_DUPLICATE = 0x01,
+	TD0_FLAG_CRC_ERROR = 0x02,
+	TD0_FLAG_DELETED   = 0x04,
+	TD0_FLAG_SKIPPED   = 0x10,
+	TD0_FLAG_IDNODATA  = 0x20,
+	TD0_FLAG_DATANOID  = 0x40,
+};
+
+class Td0ImageReader : public ImageReader
+{
+public:
+	Td0ImageReader(const ImageReaderProto& config):
+		ImageReader(config)
+	{}
+
+	std::unique_ptr<Image> readImage()
+	{
+        std::ifstream inputFile(_config.filename(), std::ios::in | std::ios::binary);
+        if (!inputFile.is_open())
+            Error() << "cannot open input file";
+
+		Bytes input;
+		input.writer() += inputFile;
+		ByteReader br(input);
+
+		uint16_t signature = br.read_be16();
+		br.skip(2); /* sequence and checksequence */
+		uint8_t version = br.read_8();
+		br.skip(2); /* data rate, drive type */
+		uint8_t stepping = br.read_8();
+		br.skip(1); /* sparse flag */
+		uint8_t sides = (br.read_8() == 1) ? 1 : 2;
+		uint16_t headerCrc = br.read_le16();
+
+		uint16_t gotCrc = crc16(0xa097, 0, input.slice(0, 10));
+		if (gotCrc != headerCrc)
+			Error() << "TD0: header checksum mismatch";
+		if (signature != 0x5444)
+			Error() << "TD0: unsupported file type (only uncompressed files are supported for now)";
+
+		std::string comment = "(no comment)";
+		if (stepping & 0x80)
+		{
+			/* Comment block */
+
+			br.skip(2); /* comment CRC */
+			uint16_t length = br.read_le16();
+			br.skip(6); /* timestamp */
+			comment = br.read(length);
+			std::replace(comment.begin(), comment.end(), '\0', '\n');
+
+			/* Strip trailing newlines */
+
+			auto nl = std::find_if(comment.rbegin(), comment.rend(),
+					[](unsigned char ch) { return !std::isspace(ch); });
+			comment.erase(nl.base(), comment.end());
+		}
+
+		std::cout << fmt::format("TD0: TeleDisk {}.{}: {}\n",
+			version / 10, version % 10, comment);
+
+		unsigned totalSize = 0;
+        std::unique_ptr<Image> image(new Image);
+		for (;;)
+		{
+			/* Read track header */
+
+			uint8_t sectorCount = br.read_8();
+			if (sectorCount == 0xff)
+				break;
+
+			uint8_t physicalCylinder = br.read_8();
+			uint8_t physicalHead = br.read_8() & 1;
+			br.skip(1); /* crc */
+
+			for (int i = 0; i < sectorCount; i++)
+			{
+				/* Read sector */
+
+				uint8_t logicalTrack = br.read_8();
+				uint8_t logicalSide = br.read_8();
+				uint8_t sectorId = br.read_8();
+				uint8_t sectorSizeEncoded = br.read_8();
+				unsigned sectorSize = 128<<sectorSizeEncoded;
+				uint8_t flags = br.read_8();
+				br.skip(1); /* CRC */
+
+				uint16_t dataSize = br.read_le16();
+				Bytes encodedData = br.read(dataSize);
+				ByteReader bre(encodedData);
+				uint8_t encoding = bre.read_8();
+
+				Bytes data;
+				if (!(flags & (TD0_FLAG_SKIPPED|TD0_FLAG_IDNODATA)))
+				{
+					switch (encoding)
+					{
+						case TD0_ENCODING_RAW:
+							data = encodedData.slice(1);
+							break;
+
+						case TD0_ENCODING_REPEATED:
+						{
+							ByteWriter bw(data);
+							while (!bre.eof())
+							{
+								uint16_t pattern = bre.read_le16();
+								uint16_t count = bre.read_le16();
+								while (count--)
+									bw.write_le16(pattern);
+							}
+							break;
+						}
+
+						case TD0_ENCODING_RLE:
+						{
+							ByteWriter bw(data);
+							while (!bre.eof())
+							{
+								uint8_t length = bre.read_8()*2;
+								if (length == 0)
+								{
+									/* Literal block */
+
+									length = bre.read_8();
+									bw += bre.read(length);
+								}
+								else
+								{
+									/* Repeated block */
+
+									uint8_t count = bre.read_8();
+									Bytes b = bre.read(length);
+									while (count--)
+										bw += b;
+								}
+							}
+							break;
+						}
+					}
+				}
+
+				const auto& sector = image->put(logicalTrack, logicalSide, sectorId);
+				sector->status = Sector::OK;
+				sector->physicalCylinder = physicalCylinder;
+				sector->physicalHead = physicalHead;
+				sector->data = data.slice(0, sectorSize);
+				totalSize += sectorSize;
+			}
+		}
+
+		image->calculateSize();
+		const Geometry& geometry = image->getGeometry();
+        std::cout << fmt::format("TD0: found {} tracks, {} sides, {} sectors, {} bytes per sector, {} kB total\n",
+                        geometry.numTracks, geometry.numSides, geometry.numSectors,
+						geometry.sectorSize,
+						totalSize / 1024);
+        return image;
+	}
+};
+
+std::unique_ptr<ImageReader> ImageReader::createTd0ImageReader(const ImageReaderProto& config)
+{
+    return std::unique_ptr<ImageReader>(new Td0ImageReader(config));
+}
+
+
+

lib/imagewriter/imagewriter.cc

@@ -47,6 +47,7 @@ void ImageWriter::updateConfigForFilename(ImageWriterProto* proto, const std::st
 		{".ldbs",     [&]() { proto->mutable_ldbs(); }},
 		{".st",       [&]() { proto->mutable_img(); }},
 		{".nsi",      [&]() { proto->mutable_nsi(); }},
+		{".xdf",      [&]() { proto->mutable_img(); }},
 	};
 
 	for (const auto& it : formats)

lib/imagewriter/imgimagewriter.cc

@@ -35,7 +35,7 @@ public:
 				ImgInputOutputProto::TrackdataProto trackdata;
 				getTrackFormat(_config.img(), trackdata, track, side);
 
-				auto sectors = getSectors(trackdata);
+				auto sectors = getSectors(trackdata, geometry.numSectors);
 				if (sectors.empty())
 				{
 					int maxSector = geometry.firstSector + geometry.numSectors - 1;
@@ -61,7 +61,7 @@ public:
 						outputFile.tellp() / 1024);
 	}
 
-	std::vector<unsigned> getSectors(const ImgInputOutputProto::TrackdataProto& trackdata)
+	std::vector<unsigned> getSectors(const ImgInputOutputProto::TrackdataProto& trackdata, unsigned numSectors)
 	{
 		std::vector<unsigned> sectors;
 		switch (trackdata.sectors_oneof_case())
@@ -76,7 +76,9 @@ public:
 			case ImgInputOutputProto::TrackdataProto::SectorsOneofCase::kSectorRange:
 			{
 				int sectorId = trackdata.sector_range().start_sector();
-				for (int i=0; i<trackdata.sector_range().sector_count(); i++)
+				if (trackdata.sector_range().has_sector_count())
+					numSectors = trackdata.sector_range().sector_count();
+				for (int i=0; i<numSectors; i++)
 					sectors.push_back(sectorId + i);
 				break;
 			}

lib/proto.cc

@@ -156,6 +156,17 @@ void setProtoFieldFromString(ProtoField& protoField, const std::string& value)
 			break;
 		}
 
+		case google::protobuf::FieldDescriptor::TYPE_ENUM:
+		{
+			const auto* enumfield = field->enum_type();
+			const auto* enumvalue = enumfield->FindValueByName(value);
+			if (!enumvalue)
+				Error() << fmt::format("unrecognised enum value '{}'", value);
+
+			reflection->SetEnum(message, field, enumvalue);
+			break;
+		}
+
 		case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
 			if (field->message_type() == RangeProto::descriptor())
 			{

lib/reader.cc

@@ -16,7 +16,6 @@
 #include "fmt/format.h"
 #include "proto.h"
 #include "lib/decoders/decoders.pb.h"
-#include "lib/data.pb.h"
 #include <iostream>
 #include <fstream>
 
@@ -27,11 +26,9 @@ static std::shared_ptr<Fluxmap> readFluxmap(FluxSource& fluxsource, unsigned cyl
 	std::cout << fmt::format("{0:>3}.{1}: ", cylinder, head) << std::flush;
 	std::shared_ptr<Fluxmap> fluxmap = fluxsource.readFlux(cylinder, head);
 	std::cout << fmt::format(
-		"{0} ms in {1} bytes\n",
+		"{0:.0} ms in {1} bytes\n",
             fluxmap->duration()/1e6,
             fluxmap->bytes());
-	if (outputFluxSink)
-		outputFluxSink->writeFlux(cylinder, head, *fluxmap);
 	return fluxmap;
 }
 
@@ -83,10 +80,13 @@ void readDiskCommand(FluxSource& fluxsource, AbstractDecoder& decoder, ImageWrit
 			auto track = std::make_unique<TrackFlux>();
 			std::set<std::shared_ptr<Sector>> track_sectors;
 			std::set<std::shared_ptr<Record>> track_records;
+			Fluxmap totalFlux;
 
 			for (int retry = config.decoder().retries(); retry >= 0; retry--)
 			{
 				auto fluxmap = readFluxmap(fluxsource, cylinder, head);
+				totalFlux.appendDesync().appendBytes(fluxmap->rawBytes());
+
 				{
 					auto trackdata = decoder.decodeToSectors(fluxmap, cylinder, head);
 
@@ -105,6 +105,7 @@ void readDiskCommand(FluxSource& fluxsource, AbstractDecoder& decoder, ImageWrit
 					track_records.insert(trackdata->records.begin(), trackdata->records.end());
 					track->trackDatas.push_back(std::move(trackdata));
 				}
+
 				auto collected_sectors = collect_sectors(track_sectors);
 				std::cout << fmt::format("{} distinct sectors; ", collected_sectors.size());
 
@@ -147,14 +148,17 @@ void readDiskCommand(FluxSource& fluxsource, AbstractDecoder& decoder, ImageWrit
 					std::cout << retry << " retries remaining" << std::endl;
 			}
 
+			if (outputFluxSink)
+				outputFluxSink->writeFlux(cylinder, head, totalFlux);
+
 			if (config.decoder().dump_records())
 			{
 				std::cout << "\nRaw (undecoded) records follow:\n\n";
 				for (const auto& record : track_records)
 				{
-					std::cout << fmt::format("I+{:.2f}us with {:.2f}us clock\n",
+					std::cout << fmt::format("I+{:.2f}us with {:.2f}us bitcell\n",
 								record->startTime / 1000.0,
-								record->clock / 1000.0);
+								record->bitcell / 1000.0);
 					hexdump(std::cout, record->rawData);
 					std::cout << std::endl;
 				}

lib/scp.h

@@ -27,17 +27,23 @@ enum
     SCP_FLAG_FOOTER     = (1<<5)
 };
 
-struct ScpTrack
+struct ScpTrackHeader
 {
     char track_id[3];      // 'TRK'
     uint8_t strack;        // SCP track number
-    struct
-    {
-        uint8_t index[4];  // time for one revolution
-        uint8_t length[4]; // number of bitcells
-        uint8_t offset[4]; // offset to bitcell data, relative to track header
-    }
-    revolution[5];
+};
+
+struct ScpTrackRevolution
+{
+    uint8_t index[4];  // time for one revolution
+    uint8_t length[4]; // number of bitcells
+    uint8_t offset[4]; // offset to bitcell data, relative to track header
+};
+
+struct ScpTrack
+{
+    ScpTrackHeader header;
+    ScpTrackRevolution revolution[5];
 };
 
 #endif

lib/sector.h

@@ -28,6 +28,7 @@ public:
 	Status status = Status::INTERNAL_ERROR;
     Fluxmap::Position position;
     nanoseconds_t clock = 0;
+    nanoseconds_t bitcell = 0;
     nanoseconds_t headerStartTime = 0;
     nanoseconds_t headerEndTime = 0;
     nanoseconds_t dataStartTime = 0;

lib/usb/fluxengineusb.cc

@@ -120,9 +120,7 @@ private:
 public:
 	int getVersion()
 	{
-		struct any_frame f;
-		f.f.type = F_FRAME_GET_VERSION_CMD;
-		f.f.size = sizeof(f);
+		struct any_frame f = { .f = {.type = F_FRAME_GET_VERSION_CMD, .size = sizeof(f)} };
 		usb_cmd_send(&f, f.f.size);
 		auto r = await_reply<struct version_frame>(F_FRAME_GET_VERSION_REPLY);
 		return r->version;
@@ -130,29 +128,29 @@ public:
 
 	void seek(int track)
 	{
-		struct seek_frame f;
-		f.f.type = F_FRAME_SEEK_CMD;
-		f.f.size = sizeof(f);
-		f.track = (uint8_t) track;
+		struct seek_frame f = {
+			.f = { .type = F_FRAME_SEEK_CMD, .size = sizeof(f) },
+			.track = (uint8_t) track
+		};
 		usb_cmd_send(&f, f.f.size);
 		await_reply<struct any_frame>(F_FRAME_SEEK_REPLY);
 	}
 
 	void recalibrate()
 	{
-		struct any_frame f;
-		f.f.type = F_FRAME_RECALIBRATE_CMD;
-		f.f.size = sizeof(f);
+		struct any_frame f = {
+			.f = { .type = F_FRAME_RECALIBRATE_CMD, .size = sizeof(f) },
+		};
 		usb_cmd_send(&f, f.f.size);
 		await_reply<struct any_frame>(F_FRAME_RECALIBRATE_REPLY);
 	}
 
 	nanoseconds_t getRotationalPeriod(int hardSectorCount)
 	{
-		struct measurespeed_frame f;
-		f.f.type = F_FRAME_MEASURE_SPEED_CMD;
-		f.f.size = sizeof(f);
-		f.hard_sector_count = (uint8_t) hardSectorCount;
+		struct measurespeed_frame f = {
+			.f = {.type = F_FRAME_MEASURE_SPEED_CMD, .size = sizeof(f)},
+			.hard_sector_count = (uint8_t) hardSectorCount,
+		};
 		usb_cmd_send(&f, f.f.size);
 
 		auto r = await_reply<struct speed_frame>(F_FRAME_MEASURE_SPEED_REPLY);
@@ -161,9 +159,7 @@ public:
 
 	void testBulkWrite()
 	{
-		struct any_frame f;
-		f.f.type = F_FRAME_BULK_WRITE_TEST_CMD;
-		f.f.size = sizeof(f);
+		struct any_frame f = { .f = {.type = F_FRAME_BULK_WRITE_TEST_CMD, .size = sizeof(f)} };
 		usb_cmd_send(&f, f.f.size);
 
 		/* These must match the device. */
@@ -202,9 +198,7 @@ public:
 
 	void testBulkRead()
 	{
-		struct any_frame f;
-		f.f.type = F_FRAME_BULK_READ_TEST_CMD;
-		f.f.size = sizeof(f);
+		struct any_frame f = { .f = {.type = F_FRAME_BULK_READ_TEST_CMD, .size = sizeof(f)} };
 		usb_cmd_send(&f, f.f.size);
 
 		/* These must match the device. */
@@ -240,11 +234,11 @@ public:
 	Bytes read(int side, bool synced, nanoseconds_t readTime,
 	           nanoseconds_t hardSectorThreshold)
 	{
-		struct read_frame f;
-		f.f.type = F_FRAME_READ_CMD;
-		f.f.size = sizeof(f);
-		f.side = (uint8_t) side;
-		f.synced = (uint8_t) synced;
+		struct read_frame f = {
+			.f = { .type = F_FRAME_READ_CMD, .size = sizeof(f) },
+			.side = (uint8_t) side,
+			.synced = (uint8_t) synced,
+		};
 		f.hardsec_threshold_ms = (hardSectorThreshold + 5e5) / 1e6; /* round to nearest ms */
 		uint16_t milliseconds = readTime / 1e6;
 		((uint8_t*)&f.milliseconds)[0] = milliseconds;
@@ -266,10 +260,10 @@ public:
 		unsigned safelen = bytes.size() & ~(FRAME_SIZE-1);
 		Bytes safeBytes = bytes.slice(0, safelen);
 
-		struct write_frame f;
-		f.f.type = F_FRAME_WRITE_CMD;
-		f.f.size = sizeof(f);
-		f.side = (uint8_t) side;
+		struct write_frame f = {
+			.f = { .type = F_FRAME_WRITE_CMD, .size = sizeof(f) },
+			.side = (uint8_t) side,
+		};
 		f.hardsec_threshold_ms = (hardSectorThreshold + 5e5) / 1e6; /* round to nearest ms */
 		((uint8_t*)&f.bytes_to_write)[0] = safelen;
 		((uint8_t*)&f.bytes_to_write)[1] = safelen >> 8;
@@ -284,10 +278,10 @@ public:
 
 	void erase(int side, nanoseconds_t hardSectorThreshold)
 	{
-		struct erase_frame f;
-		f.f.type = F_FRAME_ERASE_CMD;
-		f.f.size = sizeof(f);
-		f.side = (uint8_t) side;
+		struct erase_frame f = {
+			.f = { .type = F_FRAME_ERASE_CMD, .size = sizeof(f) },
+			.side = (uint8_t) side,
+		};
 		f.hardsec_threshold_ms = (hardSectorThreshold + 5e5) / 1e6; /* round to nearest ms */
 		usb_cmd_send(&f, f.f.size);
 
@@ -296,21 +290,21 @@ public:
 
 	void setDrive(int drive, bool high_density, int index_mode)
 	{
-		struct set_drive_frame f;
-		f.f.type = F_FRAME_SET_DRIVE_CMD;
-		f.f.size = sizeof(f);
-		f.drive = (uint8_t) drive;
-		f.high_density = high_density;
-		f.index_mode = (uint8_t) index_mode;
+		struct set_drive_frame f = {
+			.f = { .type = F_FRAME_SET_DRIVE_CMD, .size = sizeof(f) },
+			.drive = (uint8_t) drive,
+			.high_density = high_density,
+			.index_mode = (uint8_t) index_mode
+		};
 		usb_cmd_send(&f, f.f.size);
 		await_reply<struct any_frame>(F_FRAME_SET_DRIVE_REPLY);
 	}
 
 	void measureVoltages(struct voltages_frame* voltages)
 	{
-		struct any_frame f;
-		f.f.type = F_FRAME_MEASURE_VOLTAGES_CMD;
-		f.f.size = sizeof(f);
+		struct any_frame f = {
+			{ .type = F_FRAME_MEASURE_VOLTAGES_CMD, .size = sizeof(f) },
+		};
 		usb_cmd_send(&f, f.f.size);
 
 		auto convert_voltages_from_usb = [&](const struct voltages& vin, struct voltages& vout)

lib/usb/greaseweazleusb.cc

@@ -108,6 +108,11 @@
     typedef int FileHandle;
     static FileHandle open_serial_port(const std::string& name)
     {
+        #ifdef __APPLE__
+            if (name.find("/dev/tty.") != std::string::npos)
+                std::cerr << "Warning: you probably want to be using a /dev/cu.* device\n";
+        #endif
+
         int fd = open(name.c_str(), O_RDWR);
         if (fd == -1)
             Error() << fmt::format("cannot open GreaseWeazle serial port '{}': {}",
@@ -515,7 +520,7 @@ public:
     {
         do_command({ CMD_SELECT, 3, (uint8_t)drive });
         do_command({ CMD_MOTOR, 4, (uint8_t)drive, 1 });
-        do_command({ CMD_SET_PIN, 4, 2, (uint8_t)(high_density ? 0 : 1) });
+        do_command({ CMD_SET_PIN, 4, 2, (uint8_t)(high_density ? 1 : 0) });
     }
 
     void measureVoltages(struct voltages_frame* voltages)

lib/utils.cc

@@ -7,11 +7,15 @@ bool beginsWith(const std::string& value, const std::string& ending)
     return std::equal(ending.begin(), ending.end(), value.begin());
 }
 
+// Case-insensitive for endings within ASCII.
 bool endsWith(const std::string& value, const std::string& ending)
 {
     if (ending.size() > value.size())
         return false;
-    return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
+    std::string lowercase(ending.size(), 0);
+    std::transform(value.rbegin(), value.rbegin() + ending.size(), lowercase.begin(), [](unsigned char c){ return std::tolower(c); });
+    return std::equal(ending.rbegin(), ending.rend(), value.rbegin()) ||
+        std::equal(ending.rbegin(), ending.rend(), lowercase.begin());
 }
 
 

lib/utils.h

@@ -1,6 +1,8 @@
 #ifndef UTILS_H
 #define UTILS_H
 
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
+
 extern bool beginsWith(const std::string& value, const std::string& beginning);
 extern bool endsWith(const std::string& value, const std::string& ending);
 

lib/writer.cc

@@ -56,7 +56,7 @@ void writeTracksAndVerify(
 	AbstractEncoder& encoder,
 	FluxSource& fluxSource,
 	AbstractDecoder& decoder,
-	Image& image)
+	const Image& image)
 {
     std::cout << "Writing to: " << fluxSink << std::endl;
 
@@ -129,10 +129,9 @@ void fillBitmapTo(std::vector<bool>& bitmap,
 	}
 }
 
-void writeDiskCommand(ImageReader& imageReader, AbstractEncoder& encoder, FluxSink& fluxSink,
+void writeDiskCommand(const Image& image, AbstractEncoder& encoder, FluxSink& fluxSink,
 		AbstractDecoder* decoder, FluxSource* fluxSource)
 {
-	Image image = imageReader.readImage();
 	if (fluxSource && decoder)
 		writeTracksAndVerify(fluxSink, encoder, *fluxSource, *decoder, image);
 	else

lib/writer.h

@@ -1,21 +1,22 @@
-#ifndef WRITER_H
-#define WRITER_H
-
-class Fluxmap;
-class AbstractDecoder;
-class AbstractEncoder;
-class ImageReader;
-class FluxSource;
-class FluxSink;
-
-extern void writeTracks(FluxSink& fluxSink, const std::function<std::unique_ptr<Fluxmap>(int track, int side)> producer);
-
-extern void fillBitmapTo(std::vector<bool>& bitmap,
-		unsigned& cursor, unsigned terminateAt,
-		const std::vector<bool>& pattern);
-	
-extern void writeDiskCommand(ImageReader& imageReader, AbstractEncoder& encoder, FluxSink& fluxSink,
-		AbstractDecoder* decoder = nullptr, FluxSource* fluxSource = nullptr);
-extern void writeRawDiskCommand(FluxSource& fluxSource, FluxSink& fluxSink);
-
-#endif
+#ifndef WRITER_H
+#define WRITER_H
+
+class Fluxmap;
+class AbstractDecoder;
+class AbstractEncoder;
+class ImageReader;
+class FluxSource;
+class FluxSink;
+class Image;
+
+extern void writeTracks(FluxSink& fluxSink, const std::function<std::unique_ptr<Fluxmap>(int track, int side)> producer);
+
+extern void fillBitmapTo(std::vector<bool>& bitmap,
+		unsigned& cursor, unsigned terminateAt,
+		const std::vector<bool>& pattern);
+	
+extern void writeDiskCommand(const Image& image, AbstractEncoder& encoder, FluxSink& fluxSink,
+		AbstractDecoder* decoder = nullptr, FluxSource* fluxSource = nullptr);
+extern void writeRawDiskCommand(FluxSource& fluxSource, FluxSink& fluxSink);
+
+#endif

mkninja.sh

@@ -4,7 +4,7 @@ set -e
 cat <<EOF
 rule cxx
     command = $CXX $CFLAGS \$flags -I. -c -o \$out \$in -MMD -MF \$out.d
-    description = CXX \$in
+    description = CXX \$out
     depfile = \$out.d
     deps = gcc
     
@@ -36,8 +36,8 @@ rule test
     description = TEST \$in
 
 rule encodedecode
-    command = sh scripts/encodedecodetest.sh \$format \$configs > \$out
-    description = ENCODEDECODE \$format
+    command = sh scripts/encodedecodetest.sh \$format \$fluxx \$configs > \$out
+    description = ENCODEDECODE \$fluxx \$format
 
 rule strip
     command = cp -f \$in \$out && $STRIP \$out
@@ -244,14 +244,12 @@ runtest() {
     buildlibrary lib$prog.a \
         -Idep/snowhouse/include \
         -d $OBJDIR/proto/libconfig.def \
-        -d $OBJDIR/proto/libdata.def \
         "$@"
 
     buildprogram $OBJDIR/$prog \
         lib$prog.a \
         libbackend.a \
         libconfig.a \
-        libdata.a \
         libtestproto.a \
         libagg.a \
         libfmt.a
@@ -264,9 +262,14 @@ encodedecodetest() {
     format=$1
     shift
 
-    echo "build $OBJDIR/$format.encodedecode.stamp : encodedecode | fluxengine$EXTENSION scripts/encodedecodetest.sh $*"
+    echo "build $OBJDIR/$format.encodedecode.flux.stamp : encodedecode | fluxengine$EXTENSION scripts/encodedecodetest.sh $*"
     echo "    format=$format"
     echo "    configs=$*"
+    echo "    fluxx=flux"
+    echo "build $OBJDIR/$format.encodedecode.scp.stamp : encodedecode | fluxengine$EXTENSION scripts/encodedecodetest.sh $*"
+    echo "    format=$format"
+    echo "    configs=$*"
+    echo "    fluxx=scp"
 }
 
 buildlibrary libagg.a \
@@ -304,13 +307,13 @@ buildproto libconfig.a \
     lib/imagewriter/imagewriter.proto \
     lib/usb/usb.proto \
 
-buildproto libdata.a \
-    lib/data.proto
+buildproto libfl2.a \
+    lib/fl2.proto
 
 buildlibrary libbackend.a \
     -I$OBJDIR/proto \
     -d $OBJDIR/proto/libconfig.def \
-    -d $OBJDIR/proto/libdata.def \
+    -d $OBJDIR/proto/libfl2.def \
     arch/aeslanier/decoder.cc \
     arch/amiga/amiga.cc \
     arch/amiga/decoder.cc \
@@ -327,24 +330,28 @@ buildlibrary libbackend.a \
     arch/macintosh/decoder.cc \
     arch/macintosh/encoder.cc \
     arch/micropolis/decoder.cc \
+    arch/micropolis/encoder.cc \
     arch/mx/decoder.cc \
     arch/northstar/decoder.cc \
     arch/northstar/encoder.cc \
     arch/tids990/decoder.cc \
     arch/tids990/encoder.cc \
     arch/victor9k/decoder.cc \
+    arch/victor9k/encoder.cc \
     arch/zilogmcz/decoder.cc \
     lib/bitmap.cc \
     lib/bytes.cc \
     lib/crc.cc \
     lib/csvreader.cc \
     lib/decoders/decoders.cc \
+    lib/decoders/fluxdecoder.cc \
     lib/decoders/fluxmapreader.cc \
     lib/decoders/fmmfm.cc \
     lib/encoders/encoders.cc \
     lib/flags.cc \
     lib/fluxmap.cc \
     lib/fluxsink/aufluxsink.cc \
+    lib/fluxsink/fl2fluxsink.cc \
     lib/fluxsink/fluxsink.cc \
     lib/fluxsink/hardwarefluxsink.cc \
     lib/fluxsink/scpfluxsink.cc \
@@ -352,6 +359,7 @@ buildlibrary libbackend.a \
     lib/fluxsink/vcdfluxsink.cc \
     lib/fluxsource/cwffluxsource.cc \
     lib/fluxsource/erasefluxsource.cc \
+    lib/fluxsource/fl2fluxsource.cc \
     lib/fluxsource/fluxsource.cc \
     lib/fluxsource/hardwarefluxsource.cc \
     lib/fluxsource/kryoflux.cc \
@@ -369,6 +377,10 @@ buildlibrary libbackend.a \
     lib/imagereader/imgimagereader.cc \
     lib/imagereader/jv3imagereader.cc \
     lib/imagereader/nsiimagereader.cc \
+    lib/imagereader/td0imagereader.cc \
+    lib/imagereader/dimimagereader.cc \
+    lib/imagereader/fdiimagereader.cc \
+    lib/imagereader/d88imagereader.cc \
     lib/imagewriter/d64imagewriter.cc \
     lib/imagewriter/diskcopyimagewriter.cc \
     lib/imagewriter/imagewriter.cc \
@@ -410,15 +422,16 @@ FORMATS="\
     eco1 \
     f85 \
     fb100 \
+    hp9121 \
     hplif770 \
     ibm \
     ibm1200_525 \
+    ibm1232 \
     ibm1440 \
     ibm180_525 \
     ibm360_525 \
     ibm720 \
     ibm720_525 \
-    kaypro2 \
     mac400 \
     mac800 \
     micropolis \
@@ -427,7 +440,7 @@ FORMATS="\
     northstar350 \
     northstar87 \
     tids990 \
-    victor9k \
+    victor9k_ss \
     zilogmcz \
     "
 
@@ -441,7 +454,6 @@ buildmktable formats $OBJDIR/formats.cc $FORMATS
 buildlibrary libfrontend.a \
     -I$OBJDIR/proto \
     -d $OBJDIR/proto/libconfig.def \
-    -d $OBJDIR/proto/libdata.def \
     $(for a in $FORMATS; do echo $OBJDIR/proto/src/formats/$a.cc; done) \
     $OBJDIR/formats.cc \
     src/fe-analysedriveresponse.cc \
@@ -462,7 +474,7 @@ buildprogram fluxengine \
     libfrontend.a \
     libbackend.a \
     libconfig.a \
-    libdata.a \
+    libfl2.a \
     libfmt.a \
     libagg.a \
 
@@ -495,6 +507,7 @@ runtest bytes-test          tests/bytes.cc
 runtest compression-test    tests/compression.cc
 runtest csvreader-test      tests/csvreader.cc
 runtest flags-test          tests/flags.cc
+runtest fluxmapreader-test  tests/fluxmapreader.cc
 runtest fluxpattern-test    tests/fluxpattern.cc
 runtest fmmfm-test          tests/fmmfm.cc
 runtest greaseweazle-test   tests/greaseweazle.cc
@@ -502,7 +515,6 @@ runtest kryoflux-test       tests/kryoflux.cc
 runtest ldbs-test           tests/ldbs.cc
 runtest proto-test          -I$OBJDIR/proto \
                             -d $OBJDIR/proto/libconfig.def \
-                            -d $OBJDIR/proto/libdata.def \
                             -d $OBJDIR/proto/libtestproto.def \
                             tests/proto.cc \
                             $OBJDIR/proto/tests/testproto.cc
@@ -519,17 +531,18 @@ encodedecodetest atarist820
 encodedecodetest brother120
 encodedecodetest brother240
 encodedecodetest ibm1200_525
+encodedecodetest ibm1232
 encodedecodetest ibm1440
 encodedecodetest ibm180_525
 encodedecodetest ibm360_525
 encodedecodetest ibm720
 encodedecodetest ibm720_525
-encodedecodetest kaypro2
 encodedecodetest tids990
 encodedecodetest commodore1581
 encodedecodetest commodore1541 scripts/commodore1541_test.textpb
 encodedecodetest mac400 scripts/mac400_test.textpb
 encodedecodetest mac800 scripts/mac800_test.textpb
+encodedecodetest victor9k_ss
 
 # vim: sw=4 ts=4 et
 

protocol.h

@@ -88,7 +88,9 @@ enum
 enum
 {
     F_BIT_PULSE = 0x80,
-    F_BIT_INDEX = 0x40
+    F_BIT_INDEX = 0x40,
+	F_DESYNC = 0x00,
+	F_EOF = 0x100 /* synthetic, only produced by library */
 };
 
 struct frame_header

scripts/encodedecodetest.sh

@@ -1,21 +1,22 @@
 #!/bin/sh
 set -e
 
-if [ $(uname) != "Linux" ]; then
-	echo "Skipping test as not on Linux"
+if [ "$(uname)" != "Linux" -a "$(uname)" != "Darwin" ]; then
+	echo "Skipping test as not on Linux" >&2
 	exit 0
 fi
 
 tmp=/tmp/$$
 srcfile=$tmp.src.img
-fluxfile=$tmp.flux
+fluxfile=$tmp.$2
 destfile=$tmp.dest.img
 format=$1
 shift
+shift
 
 trap "rm -f $srcfile $fluxfile $destfile" EXIT
 
-dd if=/dev/urandom of=$srcfile bs=1M count=2 2>1
+dd if=/dev/urandom of=$srcfile bs=1048576 count=2 2>&1
 
 ./fluxengine write $format -i $srcfile -d $fluxfile "$@"
 ./fluxengine read $format -s $fluxfile -o $destfile "$@"

src/fe-analysedriveresponse.cc

@@ -257,10 +257,11 @@ int mainAnalyseDriveResponse(int argc, const char* argv[])
 
 			FluxmapReader fmr(inFluxmap);
 			fmr.seek((double)period*0.1); /* skip first 10% and last 10% as contains junk */
-			fmr.findEvent(F_BIT_PULSE);
+			fmr.skipToEvent(F_BIT_PULSE);
 			while (fmr.tell().ns() < ((double)period*0.9))
 			{
-				unsigned ticks = fmr.findEvent(F_BIT_PULSE);
+				unsigned ticks;
+				fmr.findEvent(F_BIT_PULSE, ticks);
 				if (ticks < numColumns)
 					frequencies[row][ticks]++;
 			}

src/fe-inspect.cc

@@ -3,6 +3,7 @@
 #include "reader.h"
 #include "fluxmap.h"
 #include "decoders/fluxmapreader.h"
+#include "decoders/fluxdecoder.h"
 #include "decoders/decoders.h"
 #include "fluxsource/fluxsource.h"
 #include "protocol.h"
@@ -91,7 +92,8 @@ static nanoseconds_t guessClock(const Fluxmap& fluxmap)
 
     while (!fr.eof())
     {
-        unsigned interval = fr.findEvent(F_BIT_PULSE);
+        unsigned interval;
+		fr.findEvent(F_BIT_PULSE, interval);
         if (interval > 0xff)
             continue;
         buckets[interval]++;
@@ -233,7 +235,9 @@ int mainInspect(int argc, const char* argv[])
 		nanoseconds_t lasttransition = 0;
 		while (!fmr.eof())
 		{
-			ticks += fmr.findEvent(F_BIT_PULSE);
+			unsigned thisTicks;
+			fmr.findEvent(F_BIT_PULSE, thisTicks);
+			ticks += thisTicks;
 
 			nanoseconds_t transition = ticks*NS_PER_TICK;
 			nanoseconds_t next;
@@ -277,6 +281,7 @@ int mainInspect(int argc, const char* argv[])
 		std::cout << fmt::format("\n\nAligned bitstream from {:.3f}ms follows:\n",
 				fmr.tell().ns() / 1000000.0);
 
+		FluxDecoder decoder(&fmr, clockPeriod, config.decoder());
 		while (!fmr.eof())
 		{
 			std::cout << fmt::format("{:06x} {: 10.3f} : ",
@@ -285,7 +290,7 @@ int mainInspect(int argc, const char* argv[])
 			{
 				if (fmr.eof())
 					break;
-				bool b = fmr.readRawBit(clockPeriod);
+				bool b = decoder.readBit();
 				std::cout << (b ? 'X' : '-');
 			}
 

src/fe-seek.cc

@@ -1,7 +1,7 @@
 #include "globals.h"
 #include "flags.h"
 #include "usb/usb.h"
-#include "fluxsource/fluxsource.cc"
+#include "fluxsource/fluxsource.h"
 #include "proto.h"
 #include "protocol.h"
 

src/fe-upgradefluxfile.cc

@@ -2,133 +2,35 @@
 #include "flags.h"
 #include "sql.h"
 #include "fluxmap.h"
+#include "writer.h"
+#include "proto.h"
+#include "lib/fluxsource/fluxsource.h"
+#include "lib/fluxsink/fluxsink.h"
+#include "lib/fluxsource/fluxsource.pb.h"
+#include "lib/fluxsink/fluxsink.pb.h"
 #include "fmt/format.h"
-
-static sqlite3* db;
-
-static void update_version_1_to_3()
-{
-    for (const auto i : sqlFindFlux(db))
-    {
-        Fluxmap after;
-        const auto before = sqlReadFlux(db, i.first, i.second);
-
-        /* Remember, before does not contain valid opcodes! */
-        unsigned pending = 0;
-        for (uint8_t b : before->rawBytes())
-        {
-            if (b < 0x80)
-            {
-                after.appendInterval(b + pending);
-                after.appendPulse();
-                pending = 0;
-            }
-            else
-                pending += 0x80;
-        }
-
-        sqlWriteFlux(db, i.first, i.second, after);
-        std::cout << '.' << std::flush;
-    }
-    std::cout << std::endl;
-}
-
-static void update_version_2_to_3()
-{
-	for (const auto i : sqlFindFlux(db))
-	{
-		Fluxmap after;
-        const auto before = sqlReadFlux(db, i.first, i.second);
-
-        /* Remember, before does not contain valid opcodes! */
-        unsigned pending = 0;
-        for (uint8_t b : before->rawBytes())
-        {
-			switch (b)
-			{
-				case 0x80: /* pulse */
-					after.appendInterval(pending);
-					after.appendPulse();
-					pending = 0;
-					break;
-
-				case 0x81: /* index */
-					after.appendInterval(pending);
-					after.appendIndex();
-					pending = 0;
-					break;
-
-				default:
-					pending += b;
-					break;
-			}
-        }
-		after.appendInterval(pending);
-
-        sqlWriteFlux(db, i.first, i.second, after);
-        std::cout << '.' << std::flush;
-    }
-    std::cout << std::endl;
-}
+#include <fstream>
 
 int mainUpgradeFluxFile(int argc, const char* argv[])
 {
     if (argc != 2)
-        Error() << "syntax: fe-upgradefluxfile <fluxfile>";
+        Error() << "syntax: fluxengine upgradefluxfile <fluxfile>";
     std::string filename = argv[1];
+	std::string newfilename = filename + ".new";
     
-    db = sqlOpen(filename, SQLITE_OPEN_READWRITE);
-    atexit([]()
-        {
-            sqlClose(db);
-        }
-    );
+    setRange(config.mutable_cylinders(), "0-79");
+    setRange(config.mutable_heads(), "0-1");
 
-    int version = sqlGetVersion(db);
-    std::cout << fmt::format("File at version {}\n", version);
-    if (version == FLUX_VERSION_CURRENT)
-    {
-        std::cout << "Up to date!\n";
-        return 0;
-    }
+	FluxSourceProto fluxSourceProto;
+	fluxSourceProto.mutable_fl2()->set_filename(filename);
 
-    if (version == FLUX_VERSION_0)
-    {
-        std::cout << "Upgrading to version 1\n";
-        sqlPrepareFlux(db);
-        sqlStmt(db, "BEGIN;");
-        sqlStmt(db,
-            "INSERT INTO zdata"
-            " SELECT track, side, data, 0 AS compression FROM rawdata;"
-        );
-        sqlStmt(db, "DROP TABLE rawdata;");
-        version = FLUX_VERSION_1;
-        sqlWriteIntProperty(db, "version", version);
-        sqlStmt(db, "COMMIT;");
-    }
+	FluxSinkProto fluxSinkProto;
+	fluxSinkProto.mutable_fl2()->set_filename(newfilename);
 
-    if (version == FLUX_VERSION_1)
-    {
-        std::cout << "Upgrading to version 3\n";
-        sqlStmt(db, "BEGIN;");
-        update_version_1_to_3();
-        version = FLUX_VERSION_3;
-        sqlWriteIntProperty(db, "version", version);
-        sqlStmt(db, "COMMIT;");
-    }
+	auto fluxSource = FluxSource::create(fluxSourceProto);
+	auto fluxSink = FluxSink::create(fluxSinkProto);
+	writeRawDiskCommand(*fluxSource, *fluxSink);
 
-	if (version == FLUX_VERSION_2)
-	{
-        std::cout << "Upgrading to version 3\n";
-        sqlStmt(db, "BEGIN;");
-        update_version_2_to_3();
-        version = FLUX_VERSION_3;
-        sqlWriteIntProperty(db, "version", version);
-        sqlStmt(db, "COMMIT;");
-	}
-
-    std::cout << "Vacuuming\n";
-    sqlStmt(db, "VACUUM;");
-    std::cout << "Upgrade done\n";
+	rename(newfilename.c_str(), filename.c_str());
     return 0;
 }

src/fe-write.cc

@@ -62,6 +62,8 @@ int mainWrite(int argc, const char* argv[])
     flags.parseFlagsWithConfigFiles(argc, argv, formats);
 
 	std::unique_ptr<ImageReader> reader(ImageReader::create(config.image_reader()));
+	std::unique_ptr<Image> image = reader->readImage();
+
 	std::unique_ptr<AbstractEncoder> encoder(AbstractEncoder::create(config.encoder()));
 	std::unique_ptr<FluxSink> fluxSink(FluxSink::create(config.flux_sink()));
 
@@ -73,7 +75,7 @@ int mainWrite(int argc, const char* argv[])
 	if (config.has_flux_source() && config.flux_source().has_drive())
 		fluxSource = FluxSource::create(config.flux_source());
 
-	writeDiskCommand(*reader, *encoder, *fluxSink, decoder.get(), fluxSource.get());
+	writeDiskCommand(*image, *encoder, *fluxSink, decoder.get(), fluxSource.get());
 
     return 0;
 }

src/formats/acornadfs.textpb

@@ -12,7 +12,9 @@ image_writer {
 }
 
 decoder {
-	ibm {}
+	ibm {
+		ignore_side_byte: true
+	}
 }
 
 cylinders {

src/formats/acorndfs.textpb

@@ -12,7 +12,9 @@ image_writer {
 }
 
 decoder {
-	ibm {}
+	ibm {
+		ignore_side_byte: true
+	}
 }
 
 cylinders {