Add some more GUI for the disk exerciser.

now contain both the raw list of sectors and a deduplicated list,
suitable for the visualiser.

.github/workflows/ccpp.yml

@@ -8,19 +8,21 @@ concurrency:
 
 jobs:
   build-linux:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
       with:
         repository: 'davidgiven/fluxengine'
         path: 'fluxengine'
+        submodules: 'true'
     - uses: actions/checkout@v4
       with:
         repository: 'davidgiven/fluxengine-testdata'
         path: 'fluxengine-testdata'
     - name: apt
       run: |
-        sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.0-gtk3-dev libfmt-dev libprotobuf-dev
+        sudo apt update
+        sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.2-dev libfmt-dev libprotobuf-dev libmagic-dev libmbedtls-dev libcurl4-openssl-dev libmagic-dev nlohmann-json3-dev libdbus-1-dev libglfw3-dev libmd4c-dev libfreetype-dev libcli11-dev libboost-regex-dev
     - name: make
       run: CXXFLAGS="-Wp,-D_GLIBCXX_ASSERTIONS" make -j`nproc` -C fluxengine
 
@@ -50,20 +52,22 @@ jobs:
   build-macos-current:
     strategy:
       matrix:
-        runs-on: [macos-13, macos-latest]
+        runs-on: [macos-15, macos-15-intel]
     runs-on: ${{ matrix.runs-on }}
     steps:
     - uses: actions/checkout@v4
       with:
         repository: 'davidgiven/fluxengine'
         path: 'fluxengine'
+        submodules: 'true'
     - uses: actions/checkout@v4
       with:
         repository: 'davidgiven/fluxengine-testdata'
         path: 'fluxengine-testdata'
     - name: brew
       run: |
-        brew install sqlite pkg-config libusb protobuf wxwidgets fmt make coreutils dylibbundler libjpeg
+        brew install sqlite pkg-config libusb protobuf wxwidgets fmt make coreutils dylibbundler libjpeg libmagic nlohmann-json cli11 boost glfw3 md4c ninja python freetype2 mbedtls
+        brew upgrade
     - name: make
       run: gmake -C fluxengine
     - name: Upload build artifacts
@@ -76,29 +80,33 @@ jobs:
 
   build-windows:
     runs-on: windows-latest
+    defaults:
+      run:
+        shell: msys2 {0}
 
     steps:
-    - name: setup WSL
+    - uses: msys2/setup-msys2@v2
-      run: |
+      with:
-        curl -L https://github.com/WhitewaterFoundry/Fedora-Remix-for-WSL/releases/download/41.0.0/Fedora-Remix-for-WSL-SL_41.0.0.0_x64_arm64.msixbundle -o fedora.msixbundle
+        msystem: mingw64
-        unzip fedora.msixbundle Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix
+        update: true
-        unzip Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix install.tar.gz
+        install: |
-        wsl --update
+          python diffutils ninja make zip
-        wsl --set-default-version 1
+        pacboy: |
-        wsl --import fedora fedora install.tar.gz
+          protobuf:p pkgconf:p curl-winssl:p file:p glfw:p mbedtls:p
-        wsl --set-default fedora
+          sqlite:p freetype:p boost:p gcc:p binutils:p nsis:p abseil-cpp:p
-        wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
-        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico'
 
-    - name: fix line endings
+    - name: debug
       run: |
-        git config --global core.autocrlf false
+        pacboy -Q --info protobuf:p
-        git config --global core.eol lf
+        cat /mingw64/lib/pkgconfig/protobuf.pc
-        
+        /mingw64/bin/pkg-config.exe protobuf --cflags
+        /mingw64/bin/pkg-config.exe protobuf --cflags --static
+
     - uses: actions/checkout@v4
       with:
         repository: 'davidgiven/fluxengine'
         path: 'fluxengine'
+        submodules: 'true'
 
     - uses: actions/checkout@v4
       with:
@@ -107,17 +115,18 @@ jobs:
 
     - name: run
       run: |
-        wsl sh -c 'make -C fluxengine BUILDTYPE=windows -j$(nproc)'
+        make -C fluxengine BUILDTYPE=windows
 
     - name: nsis
       run: |
-        wsl sh -c 'cd fluxengine && strip fluxengine.exe -o fluxengine-stripped.exe'
+        cd fluxengine
-        wsl sh -c 'cd fluxengine && strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe'
+        strip fluxengine.exe -o fluxengine-stripped.exe
-        wsl sh -c 'cd fluxengine && makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi'
+        strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe
+        makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi
 
     - name: zip
       run: |
-        wsl sh -c 'cd fluxengine && zip -9 fluxengine-windows.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex fluxengine-installer.exe'
+        cd fluxengine && zip -9 fluxengine-windows.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex fluxengine-installer.exe
 
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
         wsl --import fedora fedora install.tar.gz
         wsl --set-default fedora
         wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
-        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico'
+        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico ninja-build'
 
     - name: fix line endings
       run: |

.gitmodules

@@ -0,0 +1,36 @@
+[submodule "dep/imhex"]
+	path = dep/imhex
+	url = git@github.com:davidgiven/ImHex.git
+[submodule "dep/libwolv"]
+	path = dep/libwolv
+	url = https://github.com/WerWolv/libwolv.git
+[submodule "dep/imgui"]
+	path = dep/imgui
+	url = https://github.com/ocornut/imgui.git
+[submodule "dep/pattern-language"]
+	path = dep/pattern-language
+	url = https://github.com/WerWolv/PatternLanguage.git
+[submodule "dep/native-file-dialog"]
+	path = dep/native-file-dialog
+	url = https://github.com/btzy/nativefiledialog-extended.git
+[submodule "dep/xdgpp"]
+	path = dep/xdgpp
+	url = https://github.com/WerWolv/xdgpp.git
+[submodule "dep/libromfs"]
+	path = dep/libromfs
+	url = https://github.com/WerWolv/libromfs.git
+[submodule "dep/throwing_ptr"]
+	path = dep/throwing_ptr
+	url = https://github.com/rockdreamer/throwing_ptr.git
+[submodule "dep/lunasvg"]
+	path = dep/lunasvg
+	url = https://github.com/sammycage/lunasvg.git
+[submodule "dep/md4c"]
+	path = dep/md4c
+	url = https://github.com/mity/md4c
+[submodule "dep/nlohmann_json"]
+	path = dep/nlohmann_json
+	url = https://github.com/nlohmann/json
+[submodule "dep/cli11"]
+	path = dep/cli11
+	url = https://github.com/CLIUtils/CLI11

Makefile

@@ -8,25 +8,52 @@ ifeq ($(BUILDTYPE),)
 endif
 export BUILDTYPE
 
+OPTFLAGS = -g -O3
+
 ifeq ($(BUILDTYPE),windows)
-	MINGW = i686-w64-mingw32-
+	MINGW = x86_64-w64-mingw32-
 	CC = $(MINGW)gcc
-	CXX = $(MINGW)g++ -std=c++20
+	CXX = $(MINGW)g++
-	CFLAGS += -g -O3
+	CFLAGS += \
+		$(OPTFLAGS) \
+		-ffunction-sections \
+		-fdata-sections \
+		-Wno-attributes \
+		-Wa,-mbig-obj \
+		-static
 	CXXFLAGS += \
-		-fext-numeric-literals \
+		$(OPTFLAGS) \
+		-std=c++23 \
 		-Wno-deprecated-enum-float-conversion \
-		-Wno-deprecated-enum-enum-conversion
+		-Wno-deprecated-enum-enum-conversion \
-	LDFLAGS += -static
+		-Wno-attributes \
-	AR = $(MINGW)ar
+		-Wa,-mbig-obj \
-	PKG_CONFIG = $(MINGW)pkg-config -static
+		-static
+	LDFLAGS += -Wl,--gc-sections -static
+	AR = $(MINGW)gcc-ar
+	PKG_CONFIG = $(MINGW)pkg-config --static
 	WINDRES = $(MINGW)windres
 	WX_CONFIG = /usr/i686-w64-mingw32/sys-root/mingw/bin/wx-config-3.0 --static=yes
+	NINJA = /bin/ninja
+	PROTOC = /mingw64/bin/protoc
+	PROTOC_SEPARATOR = ;
 	EXT = .exe
+
+	AB_SANDBOX = no
 else
-	CC = gcc
+	CC = clang
-	CXX = g++ -std=c++20
+	CXX = clang++
-	CFLAGS = -g -O3 \
+	CFLAGS = \
+		$(OPTFLAGS) \
+		-I/opt/homebrew/include -I/usr/local/include \
+		-Wno-unknown-warning-option
+	CXXFLAGS = \
+		$(OPTFLAGS) \
+		-std=c++23 \
+		-fexperimental-library \
+		-I/opt/homebrew/include -I/usr/local/include \
+		-Wformat \
+		-Wformat-security \
 		-Wno-deprecated-enum-float-conversion \
 		-Wno-deprecated-enum-enum-conversion
 	LDFLAGS =
@@ -51,31 +78,33 @@ BINDIR ?= $(PREFIX)/bin
 
 # Special Windows settings.
 
-ifeq ($(OS), Windows_NT)
+#ifeq ($(OS), Windows_NT)
-	EXT ?= .exe
+#	EXT ?= .exe
-	MINGWBIN = /mingw32/bin
+#	MINGWBIN = /mingw32/bin
-	CCPREFIX = $(MINGWBIN)/
+#	CCPREFIX = $(MINGWBIN)/
-	PKG_CONFIG = $(MINGWBIN)/pkg-config
+#	PKG_CONFIG = $(MINGWBIN)/pkg-config
-	WX_CONFIG = /usr/bin/sh $(MINGWBIN)/wx-config --static=yes
+#	WX_CONFIG = /usr/bin/sh $(MINGWBIN)/wx-config --static=yes
-	PROTOC = $(MINGWBIN)/protoc
+#	PROTOC = $(MINGWBIN)/protoc
-	WINDRES = windres
+#	WINDRES = windres
-	LDFLAGS += \
+#	LDFLAGS += \
-		-static
+#		-static
-	CXXFLAGS += \
+#	CXXFLAGS += \
-		-fext-numeric-literals \
+#		-fext-numeric-literals \
-		-Wno-deprecated-enum-float-conversion \
+#		-Wno-deprecated-enum-float-conversion \
-		-Wno-deprecated-enum-enum-conversion
+#		-Wno-deprecated-enum-enum-conversion
-
+#
-	# Required to get the gcc run - time libraries on the path.
+#	# Required to get the gcc run - time libraries on the path.
-	export PATH := $(PATH):$(MINGWBIN)
+#	export PATH := $(PATH):$(MINGWBIN)
-endif
+#endif
 
 # Special OSX settings.
 
 ifeq ($(shell uname),Darwin)
 	LDFLAGS += \
 		-framework IOKit \
-		-framework Foundation 
+		-framework AppKit  \
+		-framework UniformTypeIdentifiers \
+		-framework UserNotifications
 endif
 
 .PHONY: all

arch/aeslanier/decoder.cc

@@ -36,8 +36,8 @@ public:
             decodeFmMfm(rawbits).slice(0, AESLANIER_RECORD_SIZE);
         const auto& reversed = bytes.reverseBits();
 
-        _sector->logicalTrack = reversed[1];
+        _sector->logicalCylinder = reversed[1];
-        _sector->logicalSide = 0;
+        _sector->logicalHead = 0;
         _sector->logicalSector = reversed[2];
 
         /* Check header 'checksum' (which seems far too simple to mean much). */

arch/agat/decoder.cc

@@ -59,9 +59,9 @@ public:
         if (bytes[3] != 0x5a)
             return;
 
-        _sector->logicalTrack = bytes[1] >> 1;
+        _sector->logicalCylinder = bytes[1] >> 1;
         _sector->logicalSector = bytes[2];
-        _sector->logicalSide = bytes[1] & 1;
+        _sector->logicalHead = bytes[1] & 1;
         _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
     }
 

arch/agat/encoder.cc

@@ -58,13 +58,10 @@ private:
     };
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
-        auto trackLayout = Layout::getLayoutOfTrack(
-            trackInfo->logicalTrack, trackInfo->logicalSide);
-
         double clockRateUs = _config.target_clock_period_us() / 2.0;
         int bitsPerRevolution =
             (_config.target_rotational_period_ms() * 1000.0) / clockRateUs;
@@ -80,7 +77,7 @@ public:
             writeFillerRawBytes(_config.pre_sector_gap_bytes(), 0xaaaa);
             writeRawBits(SECTOR_ID, 64);
             writeByte(0x5a);
-            writeByte((sector->logicalTrack << 1) | sector->logicalSide);
+            writeByte((sector->logicalCylinder << 1) | sector->logicalHead);
             writeByte(sector->logicalSector);
             writeByte(0x5a);
 

arch/amiga/decoder.cc

@@ -52,8 +52,8 @@ public:
         Bytes header = amigaDeinterleave(ptr, 4);
         Bytes recoveryinfo = amigaDeinterleave(ptr, 16);
 
-        _sector->logicalTrack = header[1] >> 1;
+        _sector->logicalCylinder = header[1] >> 1;
-        _sector->logicalSide = header[1] & 1;
+        _sector->logicalHead = header[1] & 1;
         _sector->logicalSector = header[2];
 
         uint32_t wantedheaderchecksum =

arch/amiga/encoder.cc

@@ -84,7 +84,7 @@ static void write_sector(std::vector<bool>& bits,
 
     checksum = 0;
     Bytes header = {0xff, /* Amiga 1.0 format byte */
-        (uint8_t)((sector->logicalTrack << 1) | sector->logicalSide),
+        (uint8_t)((sector->logicalCylinder << 1) | sector->logicalHead),
         (uint8_t)sector->logicalSector,
         (uint8_t)(AMIGA_SECTORS_PER_TRACK - sector->logicalSector)};
     write_interleaved_bytes(header);
@@ -110,7 +110,7 @@ public:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {

arch/apple2/decoder.cc

@@ -5,6 +5,7 @@
 #include "protocol.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include "arch/apple2/apple2.h"
 #include "arch/apple2/apple2.pb.h"
 #include "lib/decoders/decoders.pb.h"
@@ -93,24 +94,25 @@ public:
         ByteReader br(header);
 
         uint8_t volume = combine(br.read_be16());
-        _sector->logicalTrack = combine(br.read_be16());
+        _sector->logicalCylinder = combine(br.read_be16());
-        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalHead = _ltl->logicalHead;
         _sector->logicalSector = combine(br.read_be16());
         uint8_t checksum = combine(br.read_be16());
 
         // If the checksum is correct, upgrade the sector from MISSING
         // to DATA_MISSING in anticipation of its data record
         if (checksum ==
-            (volume ^ _sector->logicalTrack ^ _sector->logicalSector))
+            (volume ^ _sector->logicalCylinder ^ _sector->logicalSector))
             _sector->status =
                 Sector::DATA_MISSING; /* unintuitive but correct */
 
-        if (_sector->logicalSide == 1)
+        if (_sector->logicalHead == 1)
-            _sector->logicalTrack -= _config.apple2().side_one_track_offset();
+            _sector->logicalCylinder -=
+                _config.apple2().side_one_track_offset();
 
         /* Sanity check. */
 
-        if (_sector->logicalTrack > 100)
+        if (_sector->logicalCylinder > 100)
         {
             _sector->status = Sector::MISSING;
             return;

arch/apple2/encoder.cc

@@ -36,7 +36,7 @@ private:
     const Apple2EncoderProto& _config;
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -129,8 +129,8 @@ private:
             // extra padding.
             write_ff40(sector.logicalSector == 0 ? 32 : 8);
 
-            int track = sector.logicalTrack;
+            int track = sector.logicalCylinder;
-            if (sector.logicalSide == 1)
+            if (sector.logicalHead == 1)
                 track += _config.side_one_track_offset();
 
             // Write address field: APPLE2_SECTOR_RECORD + sector identifier +

arch/brother/decoder.cc

@@ -75,14 +75,14 @@ public:
         const auto& bytes = toBytes(rawbits).slice(0, 4);
 
         ByteReader br(bytes);
-        _sector->logicalTrack = decode_header_gcr(br.read_be16());
+        _sector->logicalCylinder = decode_header_gcr(br.read_be16());
         _sector->logicalSector = decode_header_gcr(br.read_be16());
 
         /* Sanity check the values read; there's no header checksum and
          * occasionally we get garbage due to bit errors. */
         if (_sector->logicalSector > 11)
             return;
-        if (_sector->logicalTrack > 79)
+        if (_sector->logicalCylinder > 79)
             return;
 
         _sector->status = Sector::DATA_MISSING;

arch/brother/encoder.cc

@@ -107,7 +107,7 @@ public:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -127,7 +127,7 @@ public:
             fillBitmapTo(bits, cursor, headerCursor, {true, false});
             write_sector_header(bits,
                 cursor,
-                sectorData->logicalTrack,
+                sectorData->logicalCylinder,
                 sectorData->logicalSector);
             fillBitmapTo(bits, cursor, dataCursor, {true, false});
             write_sector_data(bits, cursor, sectorData->data);

arch/c64/decoder.cc

@@ -74,8 +74,8 @@ public:
 
         uint8_t checksum = bytes[0];
         _sector->logicalSector = bytes[1];
-        _sector->logicalSide = 0;
+        _sector->logicalHead = 0;
-        _sector->logicalTrack = bytes[2] - 1;
+        _sector->logicalCylinder = bytes[2] - 1;
         if (checksum == xorBytes(bytes.slice(1, 4)))
             _sector->status =
                 Sector::DATA_MISSING; /* unintuitive but correct */

arch/c64/encoder.cc

@@ -155,7 +155,7 @@ public:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -178,7 +178,7 @@ public:
         else
             _formatByte1 = _formatByte2 = 0;
 
-        double clockRateUs = clockPeriodForC64Track(trackInfo->logicalTrack);
+        double clockRateUs = clockPeriodForC64Track(ltl.logicalCylinder);
         int bitsPerRevolution = 200000.0 / clockRateUs;
 
         std::vector<bool> bits(bitsPerRevolution);
@@ -245,7 +245,7 @@ private:
              *   06-07 - $0F ("off" bytes)
              */
             uint8_t encodedTrack =
-                ((sector->logicalTrack) +
+                ((sector->logicalCylinder) +
                     1); // C64 track numbering starts with 1. Fluxengine with 0.
             uint8_t encodedSector = sector->logicalSector;
             // uint8_t formatByte1 = C64_FORMAT_ID_BYTE1;

arch/f85/decoder.cc

@@ -76,8 +76,8 @@ public:
         const auto& bytes = decode(readRawBits(6 * 10));
 
         _sector->logicalSector = bytes[2];
-        _sector->logicalSide = 0;
+        _sector->logicalHead = 0;
-        _sector->logicalTrack = bytes[0];
+        _sector->logicalCylinder = bytes[0];
 
         uint16_t wantChecksum = bytes.reader().seek(4).read_be16();
         uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(0, 4));

arch/fb100/decoder.cc

@@ -126,8 +126,8 @@ public:
             return;
 
         uint8_t abssector = id[2];
-        _sector->logicalTrack = abssector >> 1;
+        _sector->logicalCylinder = abssector >> 1;
-        _sector->logicalSide = 0;
+        _sector->logicalHead = 0;
         _sector->logicalSector = abssector & 1;
         _sector->data.writer().append(id.slice(5, 12)).append(payload);
 

arch/ibm/decoder.cc

@@ -141,11 +141,10 @@ public:
         bw += decodeFmMfm(bits).slice(0, IBM_IDAM_LEN);
 
         IbmDecoderProto::TrackdataProto trackdata;
-        getTrackFormat(
+        getTrackFormat(trackdata, _ltl->logicalCylinder, _ltl->logicalHead);
-            trackdata, _sector->physicalTrack, _sector->physicalSide);
 
-        _sector->logicalTrack = br.read_8();
+        _sector->logicalCylinder = br.read_8();
-        _sector->logicalSide = br.read_8();
+        _sector->logicalHead = br.read_8();
         _sector->logicalSector = br.read_8();
         _currentSectorSize = 1 << (br.read_8() + 7);
 
@@ -156,11 +155,10 @@ public:
                 Sector::DATA_MISSING; /* correct but unintuitive */
 
         if (trackdata.ignore_side_byte())
-            _sector->logicalSide =
+            _sector->logicalHead = _ltl->logicalHead;
-                Layout::remapSidePhysicalToLogical(_sector->physicalSide);
+        _sector->logicalHead ^= trackdata.invert_side_byte();
-        _sector->logicalSide ^= trackdata.invert_side_byte();
         if (trackdata.ignore_track_byte())
-            _sector->logicalTrack = _sector->physicalTrack;
+            _sector->logicalCylinder = _ltl->logicalCylinder;
 
         for (int sector : trackdata.ignore_sector())
             if (_sector->logicalSector == sector)
@@ -209,16 +207,14 @@ public:
         _sector->status =
             (wantCrc == gotCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
 
-        auto layout = Layout::getLayoutOfTrack(
+        if (_currentSectorSize != _ltl->sectorSize)
-            _sector->logicalTrack, _sector->logicalSide);
-        if (_currentSectorSize != layout->sectorSize)
             std::cerr << fmt::format(
                 "Warning: configured sector size for t{}.h{}.s{} is {} bytes "
                 "but that seen on disk is {} bytes\n",
-                _sector->logicalTrack,
+                _sector->logicalCylinder,
-                _sector->logicalSide,
+                _sector->logicalHead,
                 _sector->logicalSector,
-                layout->sectorSize,
+                _ltl->sectorSize,
                 _currentSectorSize);
     }
 

arch/ibm/encoder.cc

@@ -107,16 +107,12 @@ private:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
         IbmEncoderProto::TrackdataProto trackdata;
-        getEncoderTrackData(
+        getEncoderTrackData(trackdata, ltl.logicalCylinder, ltl.logicalHead);
-            trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
-
-        auto trackLayout = Layout::getLayoutOfTrack(
-            trackInfo->logicalTrack, trackInfo->logicalSide);
 
         auto writeBytes = [&](const Bytes& bytes)
         {
@@ -152,7 +148,7 @@ public:
 
         uint8_t sectorSize = 0;
         {
-            int s = trackLayout->sectorSize >> 7;
+            int s = ltl.sectorSize >> 7;
             while (s > 1)
             {
                 s >>= 1;
@@ -202,9 +198,9 @@ public:
                         bw.write_8(MFM_RECORD_SEPARATOR_BYTE);
                 }
                 bw.write_8(idamUnencoded);
-                bw.write_8(sectorData->logicalTrack);
+                bw.write_8(sectorData->logicalCylinder);
                 bw.write_8(
-                    sectorData->logicalSide ^ trackdata.invert_side_byte());
+                    sectorData->logicalHead ^ trackdata.invert_side_byte());
                 bw.write_8(sectorData->logicalSector);
                 bw.write_8(sectorSize);
                 uint16_t crc = crc16(CCITT_POLY, header);
@@ -237,8 +233,7 @@ public:
                 }
                 bw.write_8(damUnencoded);
 
-                Bytes truncatedData =
+                Bytes truncatedData = sectorData->data.slice(0, ltl.sectorSize);
-                    sectorData->data.slice(0, trackLayout->sectorSize);
                 bw += truncatedData;
                 uint16_t crc = crc16(CCITT_POLY, data);
                 bw.write_be16(crc);

arch/macintosh/decoder.cc

@@ -5,6 +5,7 @@
 #include "protocol.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include "arch/macintosh/macintosh.h"
 #include "lib/core/bytes.h"
 #include "fmt/format.h"
@@ -146,7 +147,7 @@ public:
         auto header = toBytes(readRawBits(7 * 8)).slice(0, 7);
 
         uint8_t encodedTrack = decode_data_gcr(header[0]);
-        if (encodedTrack != (_sector->physicalTrack & 0x3f))
+        if (encodedTrack != (_ltl->logicalCylinder & 0x3f))
             return;
 
         uint8_t encodedSector = decode_data_gcr(header[1]);
@@ -157,8 +158,8 @@ public:
         if (encodedSector > 11)
             return;
 
-        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalCylinder = _ltl->logicalCylinder;
-        _sector->logicalSide = decode_side(encodedSide);
+        _sector->logicalHead = decode_side(encodedSide);
         _sector->logicalSector = encodedSector;
         uint8_t gotsum =
             (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;

arch/macintosh/encoder.cc

@@ -181,10 +181,10 @@ static void write_sector(std::vector<bool>& bits,
         write_bits(bits, cursor, 0xff3fcff3fcffLL, 6 * 8); /* sync */
     write_bits(bits, cursor, MAC_SECTOR_RECORD, 3 * 8);
 
-    uint8_t encodedTrack = sector->logicalTrack & 0x3f;
+    uint8_t encodedTrack = sector->logicalCylinder & 0x3f;
     uint8_t encodedSector = sector->logicalSector;
     uint8_t encodedSide =
-        encode_side(sector->logicalTrack, sector->logicalSide);
+        encode_side(sector->logicalCylinder, sector->logicalHead);
     uint8_t formatByte = MAC_FORMAT_BYTE;
     uint8_t headerChecksum =
         (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;
@@ -220,11 +220,11 @@ public:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
-        double clockRateUs = clockRateUsForTrack(trackInfo->logicalTrack);
+        double clockRateUs = clockRateUsForTrack(ltl.logicalCylinder);
         int bitsPerRevolution = 200000.0 / clockRateUs;
         std::vector<bool> bits(bitsPerRevolution);
         unsigned cursor = 0;

arch/micropolis/decoder.cc

@@ -4,6 +4,7 @@
 #include "lib/data/fluxpattern.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include "arch/micropolis/micropolis.h"
 #include "lib/core/bytes.h"
 #include "fmt/format.h"
@@ -222,14 +223,14 @@ public:
         if (syncByte != 0xFF)
             return;
 
-        _sector->logicalTrack = br.read_8();
+        _sector->logicalCylinder = br.read_8();
-        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalHead = _ltl->logicalHead;
         _sector->logicalSector = br.read_8();
         if (_sector->logicalSector > 15)
             return;
-        if (_sector->logicalTrack > 76)
+        if (_sector->logicalCylinder > 76)
             return;
-        if (_sector->logicalTrack != _sector->physicalTrack)
+        if (_sector->logicalCylinder != _ltl->logicalCylinder)
             return;
 
         br.read(10); /* OS data or padding */

arch/micropolis/encoder.cc

@@ -40,7 +40,7 @@ static void write_sector(std::vector<bool>& bits,
     {
         ByteWriter writer(sectorData);
         writer.write_8(0xff); /* Sync */
-        writer.write_8(sector->logicalTrack);
+        writer.write_8(sector->logicalCylinder);
         writer.write_8(sector->logicalSector);
         for (int i = 0; i < 10; i++)
             writer.write_8(0); /* Padding */
@@ -87,7 +87,7 @@ public:
     {
     }
 
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {

arch/mx/decoder.cc

@@ -6,6 +6,7 @@
 #include "lib/data/fluxmapreader.h"
 #include "lib/data/fluxpattern.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include <string.h>
 
 const int SECTOR_SIZE = 256;
@@ -64,8 +65,8 @@ public:
             gotChecksum += br.read_be16();
         uint16_t wantChecksum = br.read_be16();
 
-        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalCylinder = _ltl->logicalCylinder;
-        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalHead = _ltl->logicalHead;
         _sector->logicalSector = _currentSector;
         _sector->data = bytes.slice(0, SECTOR_SIZE).swab();
         _sector->status =

arch/northstar/decoder.cc

@@ -17,6 +17,7 @@
 #include "lib/data/fluxpattern.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include "arch/northstar/northstar.h"
 #include "lib/core/bytes.h"
 #include "lib/decoders/decoders.pb.h"
@@ -159,9 +160,9 @@ public:
         auto bytes = decodeFmMfm(rawbits).slice(0, recordSize);
         ByteReader br(bytes);
 
-        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalHead = _ltl->logicalHead;
         _sector->logicalSector = _hardSectorId;
-        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalCylinder = _ltl->logicalCylinder;
 
         if (headerSize == NORTHSTAR_HEADER_SIZE_DD)
         {

arch/northstar/encoder.cc

@@ -129,7 +129,7 @@ public:
     {
     }
 
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {

arch/smaky6/decoder.cc

@@ -5,6 +5,7 @@
 #include "protocol.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/sector.h"
+#include "lib/data/layout.h"
 #include "arch/smaky6/smaky6.h"
 #include "lib/core/bytes.h"
 #include "lib/core/crc.h"
@@ -129,11 +130,11 @@ public:
         uint8_t wantedChecksum = br.read_8();
         uint8_t gotChecksum = sumBytes(data) & 0xff;
 
-        if (track != _sector->physicalTrack)
+        if (track != _ltl->logicalCylinder)
             return;
 
-        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalCylinder = _ltl->physicalCylinder;
-        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalHead = _ltl->logicalHead;
         _sector->logicalSector = _sectorId;
 
         _sector->data = data;

arch/tartu/decoder.cc

@@ -43,8 +43,8 @@ public:
 
         ByteReader br(bytes);
         uint8_t track = br.read_8();
-        _sector->logicalTrack = track >> 1;
+        _sector->logicalCylinder = track >> 1;
-        _sector->logicalSide = track & 1;
+        _sector->logicalHead = track & 1;
         br.skip(1); /* seems always to be 1 */
         _sector->logicalSector = br.read_8();
         uint8_t wantChecksum = br.read_8();

arch/tartu/encoder.cc

@@ -17,7 +17,7 @@ public:
     {
     }
 
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -83,7 +83,7 @@ private:
             Bytes bytes;
             ByteWriter bw(bytes);
             bw.write_8(
-                (sectorData->logicalTrack << 1) | sectorData->logicalSide);
+                (sectorData->logicalCylinder << 1) | sectorData->logicalHead);
             bw.write_8(1);
             bw.write_8(sectorData->logicalSector);
             bw.write_8(~sumBytes(bytes.slice(0, 3)));

arch/tids990/decoder.cc

@@ -64,8 +64,8 @@ public:
         uint16_t gotChecksum =
             crc16(CCITT_POLY, bytes.slice(1, TIDS990_SECTOR_RECORD_SIZE - 3));
 
-        _sector->logicalSide = br.read_8() >> 3;
+        _sector->logicalHead = br.read_8() >> 3;
-        _sector->logicalTrack = br.read_8();
+        _sector->logicalCylinder = br.read_8();
         br.read_8(); /* number of sectors per track */
         _sector->logicalSector = br.read_8();
         br.read_be16(); /* sector size */

arch/tids990/encoder.cc

@@ -59,7 +59,7 @@ private:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -95,8 +95,8 @@ public:
 
                 writeBytes(12, 0x55);
                 bw.write_8(am1Unencoded);
-                bw.write_8(sectorData->logicalSide << 3);
+                bw.write_8(sectorData->logicalHead << 3);
-                bw.write_8(sectorData->logicalTrack);
+                bw.write_8(sectorData->logicalCylinder);
                 bw.write_8(_config.sector_count());
                 bw.write_8(sectorData->logicalSector);
                 bw.write_be16(sectorData->data.size());

arch/victor9k/decoder.cc

@@ -80,11 +80,11 @@ public:
         _sector->logicalSector = bytes[1];
         uint8_t gotChecksum = bytes[2];
 
-        _sector->logicalTrack = rawTrack & 0x7f;
+        _sector->logicalCylinder = rawTrack & 0x7f;
-        _sector->logicalSide = rawTrack >> 7;
+        _sector->logicalHead = rawTrack >> 7;
         uint8_t wantChecksum = bytes[0] + bytes[1];
-        if ((_sector->logicalSector > 20) || (_sector->logicalTrack > 85) ||
+        if ((_sector->logicalSector > 20) || (_sector->logicalCylinder > 85) ||
-            (_sector->logicalSide > 1))
+            (_sector->logicalHead > 1))
             return;
 
         if (wantChecksum == gotChecksum)

arch/victor9k/encoder.cc

@@ -112,7 +112,7 @@ static void write_sector(std::vector<bool>& bits,
     write_one_bits(bits, cursor, trackdata.pre_header_sync_bits());
     write_bits(bits, cursor, VICTOR9K_SECTOR_RECORD, 10);
 
-    uint8_t encodedTrack = sector.logicalTrack | (sector.logicalSide << 7);
+    uint8_t encodedTrack = sector.logicalCylinder | (sector.logicalHead << 7);
     uint8_t encodedSector = sector.logicalSector;
     write_bytes(bits,
         cursor,
@@ -164,13 +164,12 @@ private:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
         Victor9kEncoderProto::TrackdataProto trackdata;
-        getTrackFormat(
+        getTrackFormat(trackdata, ltl.logicalCylinder, ltl.logicalHead);
-            trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
 
         unsigned bitsPerRevolution = (trackdata.rotational_period_ms() * 1e3) /
                                      trackdata.clock_period_us();

arch/zilogmcz/decoder.cc

@@ -34,11 +34,11 @@ public:
         ByteReader br(bytes);
 
         _sector->logicalSector = br.read_8() & 0x1f;
-        _sector->logicalSide = 0;
+        _sector->logicalHead = 0;
-        _sector->logicalTrack = br.read_8() & 0x7f;
+        _sector->logicalCylinder = br.read_8() & 0x7f;
         if (_sector->logicalSector > 31)
             return;
-        if (_sector->logicalTrack > 80)
+        if (_sector->logicalCylinder > 80)
             return;
 
         _sector->data = br.read(132);

build.py

@@ -8,7 +8,7 @@ import config
 import re
 
 # Hack for building on Fedora/WSL; executables get the .exe extension,
-# build the build system detects it as Linux.
+# but the build system detects it as Linux.
 import build.toolchain
 
 toolchain.Toolchain.EXE = "$(EXT)"
@@ -93,7 +93,7 @@ else:
                     + c[1]
                     + "' '"
                     + c[2]
-                    + "' $(dir $[outs[0]]) > /dev/null"
+                    + "' $[dirname(filenameof(outs[0]))] > /dev/null"
                 ],
                 label="CORPUSTEST",
             )
@@ -104,15 +104,15 @@ export(
     name="all",
     items={
         "fluxengine$(EXT)": "src+fluxengine",
-        "fluxengine-gui$(EXT)": "src/gui",
+        "fluxengine-gui$(EXT)": "src/gui2",
         "brother120tool$(EXT)": "tools+brother120tool",
         "brother240tool$(EXT)": "tools+brother240tool",
         "upgrade-flux-file$(EXT)": "tools+upgrade-flux-file",
     }
     | (
         {
-            "FluxEngine.pkg": "src/gui+fluxengine_pkg",
+            "FluxEngine.pkg": "src/gui2+fluxengine_pkg",
-            "FluxEngine.app.zip": "src/gui+fluxengine_app_zip",
+            "FluxEngine.app.zip": "src/gui2+fluxengine_app_zip",
         }
         if config.osx
         else {}

build/ab.mk

@@ -15,16 +15,17 @@ HOSTCC ?= gcc
 HOSTCXX ?= g++
 HOSTAR ?= ar
 HOSTCFLAGS ?= -g -Og
+HOSTCXXFLAGS ?= $(HOSTCFLAGS)
 HOSTLDFLAGS ?= -g
 
 CC ?= $(HOSTCC)
 CXX ?= $(HOSTCXX)
 AR ?= $(HOSTAR)
 CFLAGS ?= $(HOSTCFLAGS)
+CXXFLAGS ?= $(CFLAGS)
 LDFLAGS ?= $(HOSTLDFLAGS)
 
-export PKG_CONFIG
+NINJA ?= ninja
-export HOST_PKG_CONFIG
 
 ifdef VERBOSE
 	hide =
@@ -63,37 +64,36 @@ EXT ?=
 
 CWD=$(shell pwd)
 
-ifeq ($(AB_ENABLE_PROGRESS_INFO),true)
+define newline
-	ifeq ($(PROGRESSINFO),)
-	# The first make invocation here has to have its output discarded or else it
-	# produces spurious 'Leaving directory' messages... don't know why.
-	rulecount := $(strip $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 > /dev/null \
-		&& $(MAKE) --no-print-directory -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l))
-	ruleindex := 1
-	PROGRESSINFO = "[$(ruleindex)/$(rulecount)]$(eval ruleindex := $(shell expr $(ruleindex) + 1)) "
-	endif
-else
-	PROGRESSINFO = ""
-endif
 
-PKG_CONFIG_HASHES = $(OBJ)/.pkg-config-hashes/target-$(word 1, $(shell $(PKG_CONFIG) --list-all | md5sum))
-HOST_PKG_CONFIG_HASHES = $(OBJ)/.pkg-config-hashes/host-$(word 1, $(shell $(HOST_PKG_CONFIG) --list-all | md5sum))
 
-$(OBJ)/build.mk : $(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES)
+endef
-$(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES) &:
-	$(hide) rm -rf $(OBJ)/.pkg-config-hashes
-	$(hide) mkdir -p $(OBJ)/.pkg-config-hashes
-	$(hide) touch $(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES)
 
-include $(OBJ)/build.mk
+define check_for_command
+  $(shell command -v $1 >/dev/null || (echo "Required command '$1' missing" >&2 && kill $$PPID))
+endef
 
-ifeq ($(OSX),yes)
+$(call check_for_command,ninja)
-	MAKEFLAGS += -r -j$(shell sysctl -n hw.logicalcpu)
+$(call check_for_command,cmp)
-else
+$(call check_for_command,$(PYTHON))
-	MAKEFLAGS += -r -j$(shell nproc)
-endif
 
-.DELETE_ON_ERROR:
+pkg-config-hash = $(shell ($(PKG_CONFIG) --list-all && $(HOST_PKG_CONFIG) --list-all) | md5sum)
+build-files = $(shell find . -name .obj -prune -o \( -name 'build.py' -a -type f \) -print) $(wildcard build/*.py) $(wildcard config.py)
+build-file-timestamps = $(shell ls -l $(build-files) | md5sum)
+
+# Wipe the build file (forcing a regeneration) if the make environment is different.
+# (Conveniently, this includes the pkg-config hash calculated above.)
+
+ignored-variables = MAKE_RESTARTS .VARIABLES MAKECMDGOALS MAKEFLAGS MFLAGS PAGER _ \
+	DESKTOP_STARTUP_ID XAUTHORITY ICEAUTHORITY SSH_AUTH_SOCK SESSION_MANAGER \
+	INVOCATION_ID SYSTEMD_EXEC_PID MANAGER_PID SSH_AGENT_PID JOURNAL_STREAM \
+	GPG_TTY WINDOWID MANAGERPID MAKE_TERMOUT MAKE_TERMERR OLDPWD
+$(shell mkdir -p $(OBJ))
+$(file >$(OBJ)/newvars.txt,$(foreach v,$(filter-out $(ignored-variables),$(.VARIABLES)),$(v)=$($(v))$(newline)))
+$(shell touch $(OBJ)/vars.txt)
+#$(shell diff -u $(OBJ)/vars.txt $(OBJ)/newvars.txt >&2)
+$(shell cmp -s $(OBJ)/newvars.txt $(OBJ)/vars.txt || (rm -f $(OBJ)/build.ninja && echo "Environment changed --- regenerating" >&2))
+$(shell mv $(OBJ)/newvars.txt $(OBJ)/vars.txt)
 
 .PHONY: update-ab
 update-ab:
@@ -107,10 +107,19 @@ clean::
 	@echo CLEAN
 	$(hide) rm -rf $(OBJ)
 
+compile_commands.json: $(OBJ)/build.ninja
+	+$(hide) $(NINJA) -f $(OBJ)/build.ninja -t compdb > $@
+
 export PYTHONHASHSEED = 1
-build-files = $(shell find . -name 'build.py') $(wildcard build/*.py) $(wildcard config.py)
+$(OBJ)/build.ninja $(OBJ)/build.targets &:
-$(OBJ)/build.mk: Makefile $(build-files) build/ab.mk
 	@echo "AB"
-	@mkdir -p $(OBJ)
+	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py \
-	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py -o $@ build.py \
+		-o $(OBJ) build.py \
-		|| rm -f $@
+		-v $(OBJ)/vars.txt \
+		|| (rm -f $@ && false)
+
+include $(OBJ)/build.targets
+.PHONY: $(ninja-targets)
+.NOTPARALLEL:
+$(ninja-targets): $(OBJ)/build.ninja
+	+$(hide) $(NINJA) -f $(OBJ)/build.ninja $@

build/ab.ninja

@@ -0,0 +1,2 @@
+rule rule
+    command = $command

build/ab.py

@@ -1,36 +1,32 @@
+from collections import namedtuple
+from copy import copy
+from importlib.machinery import SourceFileLoader, PathFinder, ModuleSpec
 from os.path import *
 from pathlib import Path
 from typing import Iterable
 import argparse
+import ast
 import builtins
-from copy import copy
 import functools
+import hashlib
 import importlib
 import importlib.util
-from importlib.machinery import (
-    SourceFileLoader,
-    PathFinder,
-    ModuleSpec,
-)
 import inspect
+import os
+import re
 import string
 import sys
-import hashlib
+import types
-import re
-import ast
-from collections import namedtuple
 
-VERBOSE_MK_FILE = False
+VERBOSE_NINJA_FILE = False
 
-verbose = False
 quiet = False
 cwdStack = [""]
 targets = {}
 unmaterialisedTargets = {}  # dict, not set, to get consistent ordering
 materialisingStack = []
 defaultGlobals = {}
-globalId = 1
+outputTargets = set()
-wordCache = {}
 
 RE_FORMAT_SPEC = re.compile(
     r"(?:(?P<fill>[\s\S])?(?P<align>[<>=^]))?"
@@ -52,6 +48,15 @@ sys.path += ["."]
 old_import = builtins.__import__
 
 
+class Environment(types.SimpleNamespace):
+    def setdefault(self, name, value):
+        if not hasattr(self, name):
+            setattr(self, name, value)
+
+
+G = Environment()
+
+
 class PathFinderImpl(PathFinder):
     def find_spec(self, fullname, path, target=None):
         # The second test here is needed for Python 3.9.
@@ -102,27 +107,88 @@ def error(message):
     raise ABException(message)
 
 
+def _undo_escaped_dollar(s, op):
+    return s.replace(f"$${op}", f"${op}")
+
+
 class BracketedFormatter(string.Formatter):
     def parse(self, format_string):
         while format_string:
-            left, *right = format_string.split("$[", 1)
+            m = re.search(f"(?:[^$]|^)()\\$\\[()", format_string)
-            if not right:
+            if not m:
-                yield (left, None, None, None)
+                yield (
+                    _undo_escaped_dollar(format_string, "["),
+                    None,
+                    None,
+                    None,
+                )
                 break
-            right = right[0]
+            left = format_string[: m.start(1)]
+            right = format_string[m.end(2) :]
 
             offset = len(right) + 1
             try:
                 ast.parse(right)
             except SyntaxError as e:
-                if not str(e).startswith("unmatched ']'"):
+                if not str(e).startswith(f"unmatched ']'"):
                     raise e
                 offset = e.offset
 
             expr = right[0 : offset - 1]
             format_string = right[offset:]
 
-            yield (left if left else None, expr, None, None)
+            yield (
+                _undo_escaped_dollar(left, "[") if left else None,
+                expr,
+                None,
+                None,
+            )
+
+
+class GlobalFormatter(string.Formatter):
+    def parse(self, format_string):
+        while format_string:
+            m = re.search(f"(?:[^$]|^)()\\$\\(([^)]*)\\)()", format_string)
+            if not m:
+                yield (
+                    format_string,
+                    None,
+                    None,
+                    None,
+                )
+                break
+            left = format_string[: m.start(1)]
+            var = m[2]
+            format_string = format_string[m.end(3) :]
+
+            yield (
+                left if left else None,
+                var,
+                None,
+                None,
+            )
+
+    def get_field(self, name, a1, a2):
+        return (
+            getattr(G, name),
+            False,
+        )
+
+    def format_field(self, value, format_spec):
+        if not value:
+            return ""
+        return str(value)
+
+
+globalFormatter = GlobalFormatter()
+
+
+def substituteGlobalVariables(value):
+    while True:
+        oldValue = value
+        value = globalFormatter.format(value)
+        if value == oldValue:
+            return _undo_escaped_dollar(value, "(")
 
 
 def Rule(func):
@@ -187,12 +253,10 @@ def _isiterable(xs):
 
 class Target:
     def __init__(self, cwd, name):
-        if verbose:
-            print("rule('%s', cwd='%s'" % (name, cwd))
         self.name = name
         self.localname = self.name.rsplit("+")[-1]
         self.traits = set()
-        self.dir = join("$(OBJ)", name)
+        self.dir = join(G.OBJ, name)
         self.ins = []
         self.outs = []
         self.deps = []
@@ -232,7 +296,8 @@ class Target:
                     [selfi.templateexpand(f) for f in filenamesof(value)]
                 )
 
-        return Formatter().format(s)
+        s = Formatter().format(s)
+        return substituteGlobalVariables(s)
 
     def materialise(self, replacing=False):
         if self not in unmaterialisedTargets:
@@ -341,10 +406,10 @@ def targetof(value, cwd=None):
             elif value.startswith("./"):
                 value = normpath(join(cwd, value))
         # Explicit directories are always raw files.
-        elif value.endswith("/"):
+        if value.endswith("/"):
             return _filetarget(value, cwd)
-        # Anything starting with a variable expansion is always a raw file.
+        # Anything in .obj is a raw file.
-        elif value.startswith("$"):
+        elif value.startswith(outputdir) or value.startswith(G.OBJ):
             return _filetarget(value, cwd)
 
         # If this is not a rule lookup...
@@ -467,78 +532,75 @@ def emit(*args, into=None):
     if into is not None:
         into += [s]
     else:
-        outputFp.write(s)
+        ninjaFp.write(s)
+
+
+def shell(*args):
+    s = "".join(args) + "\n"
+    shellFp.write(s)
 
 
 def emit_rule(self, ins, outs, cmds=[], label=None):
     name = self.name
-    fins_list = filenamesof(ins)
+    fins = [self.templateexpand(f) for f in set(filenamesof(ins))]
-    fins = set(fins_list)
+    fouts = [self.templateexpand(f) for f in filenamesof(outs)]
-    fouts = filenamesof(outs)
+
-    nonobjs = [f for f in fouts if not f.startswith("$(OBJ)")]
+    global outputTargets
+    outputTargets.update(fouts)
+    outputTargets.add(name)
 
     emit("")
-    if VERBOSE_MK_FILE:
+    if VERBOSE_NINJA_FILE:
         for k, v in self.args.items():
             emit(f"# {k} = {v}")
 
-    lines = []
-    if nonobjs:
-        emit("clean::", into=lines)
-        emit("\t$(hide) rm -f", *nonobjs, into=lines)
-
-    hashable = cmds + fins_list + fouts
-    hash = hashlib.sha1(bytes("\n".join(hashable), "utf-8")).hexdigest()
-    hashfile = join(self.dir, f"hash_{hash}")
-
-    global globalId
-    emit(".PHONY:", name, into=lines)
     if outs:
-        outsn = globalId
+        os.makedirs(self.dir, exist_ok=True)
-        globalId = globalId + 1
+        rule = []
-        insn = globalId
-        globalId = globalId + 1
 
-        emit(f"OUTS_{outsn}", "=", *fouts, into=lines)
+        if G.AB_SANDBOX == "yes":
-        emit(f"INS_{insn}", "=", *fins, into=lines)
+            sandbox = join(self.dir, "sandbox")
-        emit(name, ":", f"$(OUTS_{outsn})", into=lines)
+            emit(f"rm -rf {sandbox}", into=rule)
-        emit(hashfile, ":", into=lines)
+            emit(
-        emit(f"\t@mkdir -p {self.dir}", into=lines)
+                f"{G.PYTHON} build/_sandbox.py --link -s", sandbox, *fins, into=rule
-        emit(f"\t@touch {hashfile}", into=lines)
+            )
-        emit(
+            for c in cmds:
-            f"$(OUTS_{outsn})",
+                emit(f"(cd {sandbox} &&", c, ")", into=rule)
-            "&:" if len(fouts) > 1 else ":",
+            emit(
-            f"$(INS_{insn})",
+                f"{G.PYTHON} build/_sandbox.py --export -s",
-            hashfile,
+                sandbox,
-            into=lines,
+                *fouts,
-        )
+                into=rule,
+            )
+        else:
+            for c in cmds:
+                emit(c, into=rule)
 
+        ruletext = "".join(rule)
+        if len(ruletext) > 7000:
+            rulehash = hashlib.sha1(ruletext.encode()).hexdigest()
+
+            rulef = join(self.dir, f"rule-{rulehash}.sh")
+            with open(rulef, "wt") as fp:
+                fp.write("set -e\n")
+                fp.write(ruletext)
+
+            emit("build", *fouts, ":rule", *fins)
+            emit(" command=sh", rulef)
+        else:
+            emit("build", *fouts, ":rule", *fins)
+            emit(
+                " command=",
+                "&&".join([s.strip() for s in rule]).replace("$", "$$"),
+            )
         if label:
-            emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO)" + label, into=lines)
+            emit(" description=", label)
+        emit("build", name, ":phony", *fouts)
 
-        sandbox = join(self.dir, "sandbox")
-        emit("\t$(hide)", f"rm -rf {sandbox}", into=lines)
-        emit(
-            "\t$(hide)",
-            "$(PYTHON) build/_sandbox.py --link -s",
-            sandbox,
-            f"$(INS_{insn})",
-            into=lines,
-        )
-        for c in cmds:
-            emit(f"\t$(hide) cd {sandbox} && (", c, ")", into=lines)
-        emit(
-            "\t$(hide)",
-            "$(PYTHON) build/_sandbox.py --export -s",
-            sandbox,
-            f"$(OUTS_{outsn})",
-            into=lines,
-        )
     else:
         assert len(cmds) == 0, "rules with no outputs cannot have commands"
-        emit(name, ":", *fins, into=lines)
+        emit("build", name, ":phony", *fins)
 
-    outputFp.write("".join(lines))
     emit("")
 
 
@@ -585,47 +647,66 @@ def export(self, name=None, items: TargetsMap = {}, deps: Targets = []):
         dest = self.targetof(dest)
         outs += [dest]
 
-        destf = filenameof(dest)
+        destf = self.templateexpand(filenameof(dest))
+        outputTargets.update([destf])
 
         srcs = filenamesof([src])
         assert (
             len(srcs) == 1
         ), "a dependency of an exported file must have exactly one output file"
+        srcf = self.templateexpand(srcs[0])
 
         subrule = simplerule(
             name=f"{self.localname}/{destf}",
             cwd=self.cwd,
             ins=[srcs[0]],
             outs=[destf],
-            commands=["$(CP) -H %s %s" % (srcs[0], destf)],
+            commands=["$(CP) -H %s %s" % (srcf, destf)],
-            label="",
+            label="EXPORT",
         )
         subrule.materialise()
 
     self.ins = []
     self.outs = deps + outs
+    outputTargets.add(name)
 
     emit("")
-    emit(".PHONY:", name)
+    emit(
-    emit(name, ":", *filenamesof(outs + deps))
+        "build",
+        name,
+        ":phony",
+        *[self.templateexpand(f) for f in filenamesof(outs + deps)],
+    )
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-v", "--verbose", action="store_true")
     parser.add_argument("-q", "--quiet", action="store_true")
-    parser.add_argument("-o", "--output")
+    parser.add_argument("-v", "--varfile")
+    parser.add_argument("-o", "--outputdir")
+    parser.add_argument("-D", "--define", action="append", default=[])
     parser.add_argument("files", nargs="+")
     args = parser.parse_args()
 
-    global verbose
-    verbose = args.verbose
-
     global quiet
     quiet = args.quiet
 
-    global outputFp
+    vardefs = args.define
-    outputFp = open(args.output, "wt")
+    if args.varfile:
+        with open(args.varfile, "rt") as fp:
+            vardefs = vardefs + list(fp)
+
+    for line in vardefs:
+        if "=" in line:
+            name, value = line.split("=", 1)
+            G.setdefault(name.strip(), value.strip())
+    G.setdefault("AB_SANDBOX", "yes")
+
+    global ninjaFp, shellFp, outputdir
+    outputdir = args.outputdir
+    G.setdefault("OBJ", outputdir)
+    ninjaFp = open(outputdir + "/build.ninja", "wt")
+    ninjaFp.write(f"include build/ab.ninja\n")
 
     for k in ["Rule"]:
         defaultGlobals[k] = globals()[k]
@@ -640,7 +721,10 @@ def main():
     while unmaterialisedTargets:
         t = next(iter(unmaterialisedTargets))
         t.materialise()
-    emit("AB_LOADED = 1\n")
+
+    with open(outputdir + "/build.targets", "wt") as fp:
+        fp.write("ninja-targets =")
+        fp.write(substituteGlobalVariables(" ".join(outputTargets)))
 
 
 main()

build/c.py

@@ -7,23 +7,22 @@ from build.ab import (
     flatten,
     simplerule,
     emit,
+    G,
 )
-from build.utils import filenamesmatchingof, stripext, collectattrs
+from build.utils import stripext, collectattrs
 from build.toolchain import Toolchain, HostToolchain
 from os.path import *
 
-emit(
+if G.OSX != "yes":
-    """
+    G.STARTGROUP = "-Wl,--start-group"
-ifeq ($(OSX),no)
+    G.ENDGROUP = "-Wl,--end-group"
-STARTGROUP ?= -Wl,--start-group
+else:
-ENDGROUP ?= -Wl,--end-group
+    G.STARTGROUP = ""
-endif
+    G.ENDGROUP = ""
-"""
-)
 
 Toolchain.CC = ["$(CC) -c -o $[outs[0]] $[ins[0]] $(CFLAGS) $[cflags]"]
 Toolchain.CPP = ["$(CC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
-Toolchain.CXX = ["$(CXX) -c -o $[outs[0]] $[ins[0]] $(CFLAGS) $[cflags]"]
+Toolchain.CXX = ["$(CXX) -c -o $[outs[0]] $[ins[0]] $(CXXFLAGS) $[cflags]"]
 Toolchain.AR = ["$(AR) cqs $[outs[0]] $[ins]"]
 Toolchain.ARXX = ["$(AR) cqs $[outs[0]] $[ins]"]
 Toolchain.CLINK = [
@@ -70,13 +69,9 @@ def _toolchain_find_header_targets(deps, initial=[]):
 Toolchain.find_c_header_targets = _toolchain_find_header_targets
 
 
-HostToolchain.CC = [
+HostToolchain.CC = ["$(HOSTCC) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
-    "$(HOSTCC) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"
-]
 HostToolchain.CPP = ["$(HOSTCC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
-HostToolchain.CXX = [
+HostToolchain.CXX = ["$(HOSTCXX) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
-    "$(HOSTCXX) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"
-]
 HostToolchain.AR = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
 HostToolchain.ARXX = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
 HostToolchain.CLINK = [
@@ -102,9 +97,7 @@ def _indirect(deps, name):
     return r
 
 
-def cfileimpl(
+def cfileimpl(self, name, srcs, deps, suffix, commands, label, toolchain, cflags):
-    self, name, srcs, deps, suffix, commands, label, toolchain, cflags
-):
     outleaf = "=" + stripext(basename(filenameof(srcs[0]))) + suffix
 
     hdr_deps = toolchain.find_c_header_targets(deps)
@@ -114,9 +107,7 @@ def cfileimpl(
         if ("cheader_deps" not in d.args) and ("clibrary_deps" not in d.args)
     ]
     hdr_files = collectattrs(targets=hdr_deps, name="cheader_files")
-    cflags = collectattrs(
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
-        targets=hdr_deps, name="caller_cflags", initial=cflags
-    )
 
     t = simplerule(
         replaces=self,
@@ -194,7 +185,7 @@ def findsources(self, srcs, deps, cflags, filerule, toolchain, cwd):
     for s in flatten(srcs):
         objs += [
             filerule(
-                name=join(self.localname, _removeprefix(f, "$(OBJ)/")),
+                name=join(self.localname, _removeprefix(f, G.OBJ + "/")),
                 srcs=[f],
                 deps=deps,
                 cflags=sorted(set(cflags)),
@@ -239,9 +230,7 @@ def libraryimpl(
         i = 0
         for dest, src in hdrs.items():
             s = filenamesof([src])
-            assert (
+            assert len(s) == 1, "the target of a header must return exactly one file"
-                len(s) == 1
-            ), "the target of a header must return exactly one file"
 
             cs += [f"$(CP) $[ins[{i}]] $[outs[{i}]]"]
             outs += ["=" + dest]
@@ -431,15 +420,11 @@ def programimpl(
     label,
     filerule,
 ):
-    cfiles = findsources(
+    cfiles = findsources(self, srcs, deps, cflags, filerule, toolchain, self.cwd)
-        self, srcs, deps, cflags, filerule, toolchain, self.cwd
-    )
 
     lib_deps = toolchain.find_c_library_targets(deps)
     libs = collectattrs(targets=lib_deps, name="clibrary_files")
-    ldflags = collectattrs(
+    ldflags = collectattrs(targets=lib_deps, name="caller_ldflags", initial=ldflags)
-        targets=lib_deps, name="caller_ldflags", initial=ldflags
-    )
 
     simplerule(
         replaces=self,
@@ -558,9 +543,7 @@ def hostcxxprogram(
 
 def _cppfileimpl(self, name, srcs, deps, cflags, toolchain):
     hdr_deps = _indirect(deps, "cheader_deps")
-    cflags = collectattrs(
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
-        targets=hdr_deps, name="caller_cflags", initial=cflags
-    )
 
     simplerule(
         replaces=self,

build/pkg.py

@@ -1,4 +1,4 @@
-from build.ab import Rule, Target
+from build.ab import Rule, Target, G
 import os
 import subprocess
 
@@ -31,8 +31,8 @@ class _PkgConfig:
         return self.package_properties[p]
 
 
-TargetPkgConfig = _PkgConfig(os.getenv("PKG_CONFIG"))
+TargetPkgConfig = _PkgConfig(G.PKG_CONFIG)
-HostPkgConfig = _PkgConfig(os.getenv("HOST_PKG_CONFIG"))
+HostPkgConfig = _PkgConfig(G.HOST_PKG_CONFIG)
 
 
 def _package(self, name, package, fallback, pkgconfig):
@@ -49,9 +49,7 @@ def _package(self, name, package, fallback, pkgconfig):
         self.traits.update({"clibrary", "cxxlibrary"})
         return
 
-    assert (
+    assert fallback, f"Required package '{package}' not installed"
-        fallback
-    ), f"Required package '{package}' not installed when materialising target '$[name]'"
 
     if "cheader_deps" in fallback.args:
         self.args["cheader_deps"] = fallback.args["cheader_deps"]

build/protobuf.py

@@ -1,18 +1,16 @@
-from build.ab import Rule, Targets, emit, simplerule, filenamesof
+from build.ab import Rule, Targets, emit, simplerule, filenamesof, G
 from build.utils import filenamesmatchingof, collectattrs
 from os.path import join, abspath, dirname, relpath
 from build.pkg import has_package
 
-emit(
+G.setdefault("PROTOC", "protoc")
-    """
+G.setdefault("PROTOC_SEPARATOR", ":")
-PROTOC ?= protoc
+G.setdefault("HOSTPROTOC", "hostprotoc")
-HOSTPROTOC ?= protoc
-"""
-)
 
 assert has_package("protobuf"), "required package 'protobuf' not installed"
 
 
+
 def _getprotodeps(deps):
     r = set()
     for d in deps:
@@ -23,7 +21,7 @@ def _getprotodeps(deps):
 @Rule
 def proto(self, name, srcs: Targets = [], deps: Targets = []):
     protodeps = _getprotodeps(deps)
-    descriptorlist = ":".join(
+    descriptorlist = (G.PROTOC_SEPARATOR).join(
         [
             relpath(f, start=self.dir)
             for f in filenamesmatchingof(protodeps, "*.descriptor")
@@ -50,7 +48,7 @@ def proto(self, name, srcs: Targets = [], deps: Targets = []):
                             f"--descriptor_set_out={self.localname}.descriptor",
                         ]
                         + (
-                            [f"--descriptor_set_in={descriptorlist}"]
+                            [f"--descriptor_set_in='{descriptorlist}'"]
                             if descriptorlist
                             else []
                         )
@@ -93,7 +91,7 @@ def protocc(self, name, srcs: Targets = [], deps: Targets = []):
         outs += ["=" + cc, "=" + h]
 
     protodeps = _getprotodeps(deps + srcs)
-    descriptorlist = ":".join(
+    descriptorlist = G.PROTOC_SEPARATOR.join(
         [
             relpath(f, start=self.dir)
             for f in filenamesmatchingof(protodeps, "*.descriptor")
@@ -114,7 +112,7 @@ def protocc(self, name, srcs: Targets = [], deps: Targets = []):
                         "$(PROTOC)",
                         "--proto_path=.",
                         "--cpp_out=.",
-                        f"--descriptor_set_in={descriptorlist}",
+                        f"--descriptor_set_in='{descriptorlist}'",
                     ]
                     + protos
                 )

build/utils.py

@@ -7,10 +7,13 @@ from build.ab import (
     cwdStack,
     error,
     simplerule,
+    G
 )
 from os.path import relpath, splitext, join, basename, isfile
 from glob import iglob
 import fnmatch
+import subprocess
+import shutil
 
 
 def filenamesmatchingof(xs, pattern):
@@ -51,6 +54,16 @@ def itemsof(pattern, root=None, cwd=None):
     return result
 
 
+def does_command_exist(cmd):
+    basecmd = cmd.strip().split()[0]
+    return shutil.which(basecmd)
+
+
+def shell(cmd):
+    r = subprocess.check_output([G.SHELL, "-c", cmd])
+    return r.decode("utf-8").strip()
+
+
 @Rule
 def objectify(self, name, src: Target, symbol):
     simplerule(

build/zip.py

@@ -7,9 +7,7 @@ from build.ab import (
 
 
 @Rule
-def zip(
+def zip(self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP"):
-    self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP"
-):
     cs = ["$(PYTHON) build/_zip.py -z $[outs]"]
 
     ins = []

dep/adflib/build.py

@@ -35,7 +35,7 @@ clibrary(
         "./config.h",
         "./src/adflib.h",
     ],
-    cflags=["-Idep/adflib", "-Idep/adflib/src"],
+    cflags=["-Wno-stringop-overflow"],
     hdrs={
         "adf_blk.h": "./src/adf_blk.h",
         "adf_defs.h": "./src/adf_defs.h",

doc/building.md

@@ -204,18 +204,18 @@ install some support packages.
   - For Linux with Ubuntu/Debian:
 	`libusb-1.0-0-dev`, `libsqlite3-dev`, `zlib1g-dev`,
 	`libudev-dev`, `protobuf-compiler`, `libwxgtk3.0-gtk3-dev`,
-	`libfmt-dev`, `python3`.
+	`libfmt-dev`, `python3`. `ninja-build`
   - For Linux with Fedora/Red Hat:
     `git`, `make`, `gcc`, `gcc-c++`, `xxd`, `protobuf-compiler`,
     `protobuf-devel`, `fmt-devel`, `systemd-devel`, `wxGTK3-devel`,
-    `libsqlite3x-devel`
+    `libsqlite3x-devel`, `ninja-build`
   - For OSX with Homebrew: `libusb`, `pkg-config`, `sqlite`,
-    `protobuf`, `truncate`, `wxwidgets`, `fmt`.
+    `protobuf`, `truncate`, `wxwidgets`, `fmt`. `ninja`
   - For Windows with WSL: `protobuf-c-compiler` `protobuf-devel` `fmt-devel`
   `systemd-devel` `sqlite-devel` `wxGTK-devel` `mingw32-gcc` `mingw32-gcc-c++`
   `mingw32-zlib-static` `mingw32-protobuf-static` `mingw32-sqlite-static`
   `mingw32-wxWidgets3-static` `mingw32-libpng-static` `mingw32-libjpeg-static`
-  `mingw32-libtiff-static` `mingw32-nsis png2ico`
+  `mingw32-libtiff-static` `mingw32-nsis png2ico` `ninja-build`
 
 These lists are not necessarily exhaustive --- please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new) if I've missed

doc/disk-juku.md

@@ -0,0 +1,47 @@
+juku
+====
+## CP/M
+<!-- This file is automatically generated. Do not edit. -->
+
+Juku E5104 is an Estonian school computer from late 1980s and
+early 1990s. It was designed by EKTA in 1985, and starting
+from 1988 produced in Narva "Baltijets" factory. Arguably
+the school computer was technically outdated already when
+released, but still occupies a precious spot in the memories
+of a whole generation of Estonian IT professionals.
+
+The system uses dual 5.25 inch 2ce9
+diskette drive with regular MFM encoded DSDD. The disks have
+a sector skew factor 2 and tracks are written on one side of
+the floppy until it is full and then continued on the other
+side, starting from the outside of the disk again. This differs
+from the most common alternating sides method and somewhat
+complicates reading CP/M filesystem content with common tools.
+
+Mostly 800kB (786kB) DSDD disks were used, but there are also
+400kB (386kB) SSDD floppies in circulation.
+
+## References (all in Estonian)
+
+  - [How to read/write Juku disk images?](https://j3k.infoaed.ee/kettad/)
+  - [List of recovered Juku software](https://j3k.infoaed.ee/tarkvara-kataloog/)
+  - [System disks for E5104](https://elektroonikamuuseum.ee/juku_arvuti_tarkvara.html)
+
+## Options
+
+  - Format variants:
+      - `800`: 800kB 80-track 10-sector DSDD
+      - `400`: 400kB 80-track 10-sector SSDD
+
+## Examples
+
+To read:
+
+  - `fluxengine read -c juku --800 -s drive:0 -o image.juk`
+  - `fluxengine read -c juku --400 -s drive:0 -o image.juk`
+
+To write:
+
+  - `fluxengine write -c juku --800 -d drive:0 -i image.juk`
+  - `fluxengine write -c juku --400 -d drive:0 -i image.juk`
+

doc/disk-micropolis.md

@@ -52,7 +52,7 @@ need to apply extra options to change the format if desired.
 
 ## Options
 
-  - :
+  - $format:
       - `143`: 143kB 5.25" SSDD hard-sectored; Micropolis MetaFloppy Mod I
       - `287`: 287kB 5.25" DSDD hard-sectored; Micropolis MetaFloppy Mod I
       - `315`: 315kB 5.25" SSDD hard-sectored; Micropolis MetaFloppy Mod II

doc/disk-ti99.md

@@ -0,0 +1,26 @@
+ti99
+====
+## 90kB 35-track SSSD
+<!-- This file is automatically generated. Do not edit. -->
+
+The TI-99 was a deeply weird microcomputer from 1981, whose main claim to fame
+was being built around a 16-bit TMS9900 CPU --- and also having only 256 bytes
+of system RAM, with an additional 16kB of video RAM, requiring the BASIC to
+store the user's program in video RAM.
+
+It had an optional rack-mount expansion system with an optional disk drive. This
+was controlled by a standard FD1771 or FD179x chip, meaning a relatively normal
+IBM-scheme disk format of 35 tracks containing nine 256-byte sectors.
+
+FluxEngine can read these.
+
+## Options
+
+(no options)
+
+## Examples
+
+To read:
+
+  - `fluxengine read -c ti99 -s drive:0 -o ti99.img`
+

doc/using.md

@@ -385,9 +385,8 @@ disks, and have different magnetic properties. 3.5" drives can usually
 autodetect what kind of medium is inserted into the drive based on the hole in
 the disk casing, but 5.25" drives can't. As a result, you need to explicitly
 tell FluxEngine on the command line whether you're using a high density disk or
-not with the `--drive.high_density` configuration setting.
+not with the `--hd` configuration setting.  **If you don't do this, your disks
-**If you don't do this, your disks may not read correctly and will _certainly_
+may not read correctly and will _certainly_ fail to write correctly.**
-fail to write correctly.**
 
 You can distinguish high density 5.25" floppies from the presence of a
 traction ring around the hole in the middle of the disk; if the ring is not

extras/build.py

@@ -4,6 +4,7 @@ from build.c import clibrary
 from build.zip import zip
 from glob import glob
 from os.path import *
+import config
 
 icons = ["fluxfile", "hardware", "icon", "imagefile"]
 
@@ -17,37 +18,37 @@ clibrary(
     },
 )
 
-simplerule(
+if config.osx:
-    name="fluxengine_icns",
+    simplerule(
-    ins=["./icon.png"],
+        name="fluxengine_icns",
-    outs=["=fluxengine.icns"],
+        ins=["./icon.png"],
-    commands=[
+        outs=["=fluxengine.icns"],
-        "mkdir -p fluxengine.iconset",
+        commands=[
-        "sips -z 64 64 $[ins[0]] --out fluxengine.iconset/icon_32x32@2x.png > /dev/null",
+            "mkdir -p fluxengine.iconset",
-        "iconutil -c icns -o $[outs[0]] fluxengine.iconset",
+            "sips -z 64 64 $[ins[0]] --out fluxengine.iconset/icon_32x32@2x.png > /dev/null",
-    ],
+            "iconutil -c icns -o $[outs[0]] fluxengine.iconset",
-    label="ICONSET",
+        ],
-)
+        label="ICONSET",
-
+    )
-simplerule(
+
-    name="fluxengine_ico",
+    template_files = [
-    ins=["./icon.png"],
+        f
-    outs=["=fluxengine.ico"],
+        for f in glob("**", recursive=True, root_dir="extras/FluxEngine.app.template")
-    commands=["png2ico $[outs[0]] $[ins[0]]"],
+        if isfile(join("extras/FluxEngine.app.template", f))
-    label="MAKEICON",
+    ]
-)
+    zip(
-
+        name="fluxengine_template",
-template_files = [
+        items={
-    f
+            join("FluxEngine.app", k): join("extras/FluxEngine.app.template", k)
-    for f in glob(
+            for k in template_files
-        "**", recursive=True, root_dir="extras/FluxEngine.app.template"
+        },
+    )
+
+if config.windows:
+    simplerule(
+        name="fluxengine_ico",
+        ins=["./icon.png"],
+        outs=["=fluxengine.ico"],
+        commands=["png2ico $[outs[0]] $[ins[0]]"],
+        label="MAKEICON",
     )
-    if isfile(join("extras/FluxEngine.app.template", f))
-]
-zip(
-    name="fluxengine_template",
-    items={
-        join("FluxEngine.app", k): join("extras/FluxEngine.app.template", k)
-        for k in template_files
-    },
-)

lib/algorithms/readerwriter.cc

@@ -84,16 +84,12 @@ void renderLogMessage(
 void renderLogMessage(
     LogRenderer& r, std::shared_ptr<const TrackReadLogMessage> m)
 {
-    const auto& track = *m->track;
-
     std::set<std::shared_ptr<const Sector>> rawSectors;
     std::set<std::shared_ptr<const Record>> rawRecords;
-    for (const auto& trackDataFlux : track.trackDatas)
+    for (const auto& track : m->tracks)
     {
-        rawSectors.insert(
+        rawSectors.insert(track->allSectors.begin(), track->allSectors.end());
-            trackDataFlux->sectors.begin(), trackDataFlux->sectors.end());
+        rawRecords.insert(track->records.begin(), track->records.end());
-        rawRecords.insert(
-            trackDataFlux->records.begin(), trackDataFlux->records.end());
     }
 
     nanoseconds_t clock = 0;
@@ -114,22 +110,22 @@ void renderLogMessage(
     r.newline().add("sectors:");
 
     std::vector<std::shared_ptr<const Sector>> sectors(
-        track.sectors.begin(), track.sectors.end());
+        m->sectors.begin(), m->sectors.end());
     std::sort(sectors.begin(), sectors.end(), sectorPointerSortPredicate);
 
-    for (const auto& sector : sectors)
+    for (const auto& sector : rawSectors)
         r.add(fmt::format("{}.{}.{}{}",
-            sector->logicalTrack,
+            sector->logicalCylinder,
-            sector->logicalSide,
+            sector->logicalHead,
             sector->logicalSector,
             Sector::statusToChar(sector->status)));
 
     int size = 0;
     std::set<std::pair<int, int>> track_ids;
-    for (const auto& sector : m->track->sectors)
+    for (const auto& sector : m->sectors)
     {
         track_ids.insert(
-            std::make_pair(sector->logicalTrack, sector->logicalSide));
+            std::make_pair(sector->logicalCylinder, sector->logicalHead));
         size += sector->data.size();
     }
 
@@ -184,12 +180,16 @@ private:
         _cache;
 };
 
-void measureDiskRotation()
+static nanoseconds_t getRotationalPeriodFromConfig()
+{
+    return globalConfig()->drive().rotational_period_ms() * 1e6;
+}
+
+static nanoseconds_t measureDiskRotation()
 {
     log(BeginSpeedOperationLogMessage());
 
-    nanoseconds_t oneRevolution =
+    nanoseconds_t oneRevolution = getRotationalPeriodFromConfig();
-        globalConfig()->drive().rotational_period_ms() * 1e6;
     if (oneRevolution == 0)
     {
         usbSetDrive(globalConfig()->drive().drive(),
@@ -224,22 +224,24 @@ void measureDiskRotation()
         error("Failed\nIs a disk in the drive?");
 
     log(EndSpeedOperationLogMessage{oneRevolution});
+    return oneRevolution;
 }
 
 /* Given a set of sectors, deduplicates them sensibly (e.g. if there is a good
  * and bad version of the same sector, the bad version is dropped). */
 
-static std::set<std::shared_ptr<const Sector>> collectSectors(
+static std::vector<std::shared_ptr<const Sector>> collectSectors(
-    std::set<std::shared_ptr<const Sector>>& track_sectors,
+    std::vector<std::shared_ptr<const Sector>>& trackSectors,
     bool collapse_conflicts = true)
 {
     typedef std::tuple<unsigned, unsigned, unsigned> key_t;
     std::multimap<key_t, std::shared_ptr<const Sector>> sectors;
 
-    for (const auto& sector : track_sectors)
+    for (const auto& sector : trackSectors)
     {
-        key_t sectorid = {
+        key_t sectorid = {sector->logicalCylinder,
-            sector->logicalTrack, sector->logicalSide, sector->logicalSector};
+            sector->logicalHead,
+            sector->logicalSector};
         sectors.insert({sectorid, sector});
     }
 
@@ -281,42 +283,52 @@ static std::set<std::shared_ptr<const Sector>> collectSectors(
         sector_set.insert(new_sector);
         it = ub;
     }
-    return sector_set;
+    return sector_set | std::ranges::to<std::vector>();
 }
 
-BadSectorsState combineRecordAndSectors(TrackFlux& trackFlux,
+struct CombinationResult
-    Decoder& decoder,
-    std::shared_ptr<const TrackInfo>& trackLayout)
 {
-    std::set<std::shared_ptr<const Sector>> track_sectors;
+    BadSectorsState result;
+    std::vector<std::shared_ptr<const Sector>> sectors;
+};
+
+static CombinationResult combineRecordAndSectors(
+    std::vector<std::shared_ptr<const Track>>& tracks,
+    Decoder& decoder,
+    const std::shared_ptr<const LogicalTrackLayout>& ltl)
+{
+    CombinationResult cr = {HAS_NO_BAD_SECTORS};
+    std::vector<std::shared_ptr<const Sector>> track_sectors;
 
     /* Add the sectors which were there. */
 
-    for (auto& trackdataflux : trackFlux.trackDatas)
+    for (auto& track : tracks)
-        track_sectors.insert(
+        for (auto& sector : track->allSectors)
-            trackdataflux->sectors.begin(), trackdataflux->sectors.end());
+            track_sectors.push_back(sector);
 
     /* Add the sectors which should be there. */
 
-    for (unsigned sectorId : trackLayout->naturalSectorOrder)
+    for (unsigned sectorId : ltl->diskSectorOrder)
     {
-        auto sector = std::make_shared<Sector>(LogicalLocation{
+        auto sector = std::make_shared<Sector>(
-            trackLayout->logicalTrack, trackLayout->logicalSide, sectorId});
+            LogicalLocation{ltl->logicalCylinder, ltl->logicalHead, sectorId});
 
         sector->status = Sector::MISSING;
-        track_sectors.insert(sector);
+        sector->physicalLocation = std::make_optional(
+            CylinderHead(ltl->physicalCylinder, ltl->physicalHead));
+        track_sectors.push_back(sector);
     }
 
     /* Deduplicate. */
 
-    trackFlux.sectors = collectSectors(track_sectors);
+    cr.sectors = collectSectors(track_sectors);
-    if (trackFlux.sectors.empty())
+    if (cr.sectors.empty())
-        return HAS_BAD_SECTORS;
+        cr.result = HAS_BAD_SECTORS;
-    for (const auto& sector : trackFlux.sectors)
+    for (const auto& sector : cr.sectors)
         if (sector->status != Sector::OK)
-            return HAS_BAD_SECTORS;
+            cr.result = HAS_BAD_SECTORS;
 
-    return HAS_NO_BAD_SECTORS;
+    return cr;
 }
 
 static void adjustTrackOnError(FluxSource& fluxSource, int baseTrack)
@@ -339,179 +351,226 @@ static void adjustTrackOnError(FluxSource& fluxSource, int baseTrack)
     }
 }
 
-ReadResult readGroup(FluxSourceIteratorHolder& fluxSourceIteratorHolder,
+struct ReadGroupResult
-    std::shared_ptr<const TrackInfo>& trackInfo,
+{
-    TrackFlux& trackFlux,
+    ReadResult result;
+    std::vector<std::shared_ptr<const Sector>> combinedSectors;
+};
+
+static ReadGroupResult readGroup(const DiskLayout& diskLayout,
+    FluxSourceIteratorHolder& fluxSourceIteratorHolder,
+    const std::shared_ptr<const LogicalTrackLayout>& ltl,
+    std::vector<std::shared_ptr<const Track>>& tracks,
     Decoder& decoder)
 {
-    ReadResult result = BAD_AND_CAN_NOT_RETRY;
+    ReadGroupResult rgr = {BAD_AND_CAN_NOT_RETRY};
+
+    /* Before doing the read, look to see if we already have the necessary
+     * sectors. */
 
-    for (unsigned offset = 0; offset < trackInfo->groupSize;
-        offset += Layout::getHeadWidth())
     {
-        log(BeginReadOperationLogMessage{
+        auto [result, sectors] = combineRecordAndSectors(tracks, decoder, ltl);
-            trackInfo->physicalTrack + offset, trackInfo->physicalSide});
+        rgr.combinedSectors = sectors;
+        if (result == HAS_NO_BAD_SECTORS)
+        {
+            /* We have all necessary sectors, so can stop here. */
+            rgr.result = GOOD_READ;
+            if (globalConfig()->decoder().skip_unnecessary_tracks())
+                return rgr;
+        }
+    }
+
+    for (unsigned offset = 0; offset < ltl->groupSize;
+        offset += diskLayout.headWidth)
+    {
+        unsigned physicalCylinder = ltl->physicalCylinder + offset;
+        unsigned physicalHead = ltl->physicalHead;
+        auto& ptl = diskLayout.layoutByPhysicalLocation.at(
+            {physicalCylinder, physicalHead});
+
+        /* Do the physical read. */
+
+        log(BeginReadOperationLogMessage{physicalCylinder, physicalHead});
 
         auto& fluxSourceIterator = fluxSourceIteratorHolder.getIterator(
-            trackInfo->physicalTrack + offset, trackInfo->physicalSide);
+            physicalCylinder, physicalHead);
         if (!fluxSourceIterator.hasNext())
             continue;
 
-        std::shared_ptr<const Fluxmap> fluxmap = fluxSourceIterator.next();
+        auto fluxmap = fluxSourceIterator.next();
-        // ->rescale(
-        //     1.0 / globalConfig()->flux_source().rescale());
         log(EndReadOperationLogMessage());
         log("{0} ms in {1} bytes",
             (int)(fluxmap->duration() / 1e6),
             fluxmap->bytes());
 
-        auto trackdataflux = decoder.decodeToSectors(fluxmap, trackInfo);
+        auto flux = decoder.decodeToSectors(std::move(fluxmap), ptl);
-        trackFlux.trackDatas.push_back(trackdataflux);
+        flux->normalisedSectors = collectSectors(flux->allSectors);
-        if (combineRecordAndSectors(trackFlux, decoder, trackInfo) ==
+        tracks.push_back(flux);
-            HAS_NO_BAD_SECTORS)
+
+        /* Decode what we've got so far. */
+
+        auto [result, sectors] = combineRecordAndSectors(tracks, decoder, ltl);
+        rgr.combinedSectors = sectors;
+        if (result == HAS_NO_BAD_SECTORS)
         {
-            result = GOOD_READ;
+            /* We have all necessary sectors, so can stop here. */
+            rgr.result = GOOD_READ;
             if (globalConfig()->decoder().skip_unnecessary_tracks())
-                return result;
+                break;
         }
         else if (fluxSourceIterator.hasNext())
-            result = BAD_AND_CAN_RETRY;
+        {
+            /* The flux source claims it can do more reads, so mark this
+             * group as being retryable. */
+            rgr.result = BAD_AND_CAN_RETRY;
+        }
     }
 
-    return result;
+    return rgr;
 }
 
-void writeTracks(FluxSink& fluxSink,
+void writeTracks(const DiskLayout& diskLayout,
+
+    FluxSinkFactory& fluxSinkFactory,
     std::function<std::unique_ptr<const Fluxmap>(
-        std::shared_ptr<const TrackInfo>& trackInfo)> producer,
+        const std::shared_ptr<const LogicalTrackLayout>&)> producer,
-    std::function<bool(std::shared_ptr<const TrackInfo>& trackInfo)> verifier,
+    std::function<bool(const std::shared_ptr<const LogicalTrackLayout>&)>
-    std::vector<std::shared_ptr<const TrackInfo>>& trackInfos)
+        verifier,
+    const std::vector<CylinderHead>& logicalLocations)
 {
     log(BeginOperationLogMessage{"Encoding and writing to disk"});
 
-    if (fluxSink.isHardware())
+    if (fluxSinkFactory.isHardware())
         measureDiskRotation();
-    int index = 0;
-    for (auto& trackInfo : trackInfos)
     {
-        log(OperationProgressLogMessage{
+        auto fluxSink = fluxSinkFactory.create();
-            index * 100 / (unsigned)trackInfos.size()});
+        int index = 0;
-        index++;
+        for (auto& ch : logicalLocations)
-
-        testForEmergencyStop();
-
-        int retriesRemaining = globalConfig()->decoder().retries();
-        for (;;)
         {
-            for (int offset = 0; offset < trackInfo->groupSize;
+            log(OperationProgressLogMessage{
-                offset += Layout::getHeadWidth())
+                index * 100 / (unsigned)logicalLocations.size()});
+            index++;
+
+            testForEmergencyStop();
+
+            const auto& ltl = diskLayout.layoutByLogicalLocation.at(ch);
+            int retriesRemaining = globalConfig()->decoder().retries();
+            for (;;)
             {
-                unsigned physicalTrack = trackInfo->physicalTrack + offset;
+                for (int offset = 0; offset < ltl->groupSize;
-
+                    offset += diskLayout.headWidth)
-                log(BeginWriteOperationLogMessage{
-                    physicalTrack, trackInfo->physicalSide});
-
-                if (offset == globalConfig()->drive().group_offset())
                 {
-                    auto fluxmap = producer(trackInfo);
+                    unsigned physicalCylinder = ltl->physicalCylinder + offset;
-                    if (!fluxmap)
+                    unsigned physicalHead = ltl->physicalHead;
-                        goto erase;
 
-                    fluxSink.writeFlux(
+                    log(BeginWriteOperationLogMessage{
-                        physicalTrack, trackInfo->physicalSide, *fluxmap);
+                        physicalCylinder, ltl->physicalHead});
-                    log("writing {0} ms in {1} bytes",
-                        int(fluxmap->duration() / 1e6),
-                        fluxmap->bytes());
-                }
-                else
-                {
-                erase:
-                    /* Erase this track rather than writing. */
 
-                    Fluxmap blank;
+                    if (offset == globalConfig()->drive().group_offset())
-                    fluxSink.writeFlux(
+                    {
-                        physicalTrack, trackInfo->physicalSide, blank);
+                        auto fluxmap = producer(ltl);
-                    log("erased");
+                        if (!fluxmap)
+                            goto erase;
+
+                        fluxSink->addFlux(
+                            physicalCylinder, physicalHead, *fluxmap);
+                        log("writing {0} ms in {1} bytes",
+                            int(fluxmap->duration() / 1e6),
+                            fluxmap->bytes());
+                    }
+                    else
+                    {
+                    erase:
+                        /* Erase this track rather than writing. */
+
+                        Fluxmap blank;
+                        fluxSink->addFlux(
+                            physicalCylinder, physicalHead, blank);
+                        log("erased");
+                    }
+
+                    log(EndWriteOperationLogMessage());
                 }
 
-                log(EndWriteOperationLogMessage());
+                if (verifier(ltl))
+                    break;
+
+                if (retriesRemaining == 0)
+                    error("fatal error on write");
+
+                log("retrying; {} retries remaining", retriesRemaining);
+                retriesRemaining--;
             }
-
-            if (verifier(trackInfo))
-                break;
-
-            if (retriesRemaining == 0)
-                error("fatal error on write");
-
-            log("retrying; {} retries remaining", retriesRemaining);
-            retriesRemaining--;
         }
     }
 
     log(EndOperationLogMessage{"Write complete"});
 }
 
-void writeTracks(FluxSink& fluxSink,
+void writeTracks(const DiskLayout& diskLayout,
+    FluxSinkFactory& fluxSinkFactory,
     Encoder& encoder,
     const Image& image,
-    std::vector<std::shared_ptr<const TrackInfo>>& trackInfos)
+    const std::vector<CylinderHead>& chs)
 {
     writeTracks(
-        fluxSink,
+        diskLayout,
-        [&](std::shared_ptr<const TrackInfo>& trackInfo)
+        fluxSinkFactory,
+        [&](const std::shared_ptr<const LogicalTrackLayout>& ltl)
         {
-            auto sectors = encoder.collectSectors(trackInfo, image);
+            auto sectors = encoder.collectSectors(*ltl, image);
-            return encoder.encode(trackInfo, sectors, image);
+            return encoder.encode(*ltl, sectors, image);
         },
         [](const auto&)
         {
             return true;
         },
-        trackInfos);
+        chs);
 }
 
-void writeTracksAndVerify(FluxSink& fluxSink,
+void writeTracksAndVerify(const DiskLayout& diskLayout,
+    FluxSinkFactory& fluxSinkFactory,
     Encoder& encoder,
     FluxSource& fluxSource,
     Decoder& decoder,
     const Image& image,
-    std::vector<std::shared_ptr<const TrackInfo>>& trackInfos)
+    const std::vector<CylinderHead>& chs)
 {
     writeTracks(
-        fluxSink,
+        diskLayout,
-        [&](std::shared_ptr<const TrackInfo>& trackInfo)
+        fluxSinkFactory,
+        [&](const std::shared_ptr<const LogicalTrackLayout>& ltl)
         {
-            auto sectors = encoder.collectSectors(trackInfo, image);
+            auto sectors = encoder.collectSectors(*ltl, image);
-            return encoder.encode(trackInfo, sectors, image);
+            return encoder.encode(*ltl, sectors, image);
         },
-        [&](std::shared_ptr<const TrackInfo>& trackInfo)
+        [&](const std::shared_ptr<const LogicalTrackLayout>& ltl)
         {
-            auto trackFlux = std::make_shared<TrackFlux>();
-            trackFlux->trackInfo = trackInfo;
             FluxSourceIteratorHolder fluxSourceIteratorHolder(fluxSource);
-            auto result = readGroup(
+            std::vector<std::shared_ptr<const Track>> tracks;
-                fluxSourceIteratorHolder, trackInfo, *trackFlux, decoder);
+            auto [result, sectors] = readGroup(
-            log(TrackReadLogMessage{trackFlux});
+                diskLayout, fluxSourceIteratorHolder, ltl, tracks, decoder);
+            log(TrackReadLogMessage{tracks, sectors});
 
             if (result != GOOD_READ)
             {
-                adjustTrackOnError(fluxSource, trackInfo->physicalTrack);
+                adjustTrackOnError(fluxSource, ltl->physicalCylinder);
                 log("bad read");
                 return false;
             }
 
             Image wanted;
-            for (const auto& sector : encoder.collectSectors(trackInfo, image))
+            for (const auto& sector : encoder.collectSectors(*ltl, image))
                 wanted
-                    .put(sector->logicalTrack,
+                    .put(sector->logicalCylinder,
-                        sector->logicalSide,
+                        sector->logicalHead,
                         sector->logicalSector)
                     ->data = sector->data;
 
-            for (const auto& sector : trackFlux->sectors)
+            for (const auto& sector : sectors)
             {
-                const auto s = wanted.get(sector->logicalTrack,
+                const auto s = wanted.get(sector->logicalCylinder,
-                    sector->logicalSide,
+                    sector->logicalHead,
                     sector->logicalSector);
                 if (!s)
                 {
@@ -523,8 +582,8 @@ void writeTracksAndVerify(FluxSink& fluxSink,
                     log("data mismatch on verify");
                     return false;
                 }
-                wanted.erase(sector->logicalTrack,
+                wanted.erase(sector->logicalCylinder,
-                    sector->logicalSide,
+                    sector->logicalHead,
                     sector->logicalSector);
             }
             if (!wanted.empty())
@@ -534,60 +593,75 @@ void writeTracksAndVerify(FluxSink& fluxSink,
             }
             return true;
         },
-        trackInfos);
+        chs);
 }
 
-void writeDiskCommand(const Image& image,
+void writeDiskCommand(const DiskLayout& diskLayout,
+    const Image& image,
     Encoder& encoder,
-    FluxSink& fluxSink,
+    FluxSinkFactory& fluxSinkFactory,
     Decoder* decoder,
     FluxSource* fluxSource,
     const std::vector<CylinderHead>& physicalLocations)
 {
-    auto trackinfos = Layout::getLayoutOfTracksPhysical(physicalLocations);
+    auto chs = std::ranges::views::keys(diskLayout.layoutByLogicalLocation) |
+               std::ranges::to<std::vector>();
     if (fluxSource && decoder)
-        writeTracksAndVerify(
+        writeTracksAndVerify(diskLayout,
-            fluxSink, encoder, *fluxSource, *decoder, image, trackinfos);
+            fluxSinkFactory,
+            encoder,
+            *fluxSource,
+            *decoder,
+            image,
+            chs);
     else
-        writeTracks(fluxSink, encoder, image, trackinfos);
+        writeTracks(diskLayout, fluxSinkFactory, encoder, image, chs);
 }
 
-void writeDiskCommand(const Image& image,
+void writeDiskCommand(const DiskLayout& diskLayout,
+    const Image& image,
     Encoder& encoder,
-    FluxSink& fluxSink,
+    FluxSinkFactory& fluxSinkFactory,
     Decoder* decoder,
     FluxSource* fluxSource)
 {
-    auto locations = Layout::computePhysicalLocations();
+    writeDiskCommand(diskLayout,
-    writeDiskCommand(image, encoder, fluxSink, decoder, fluxSource, locations);
+        image,
+        encoder,
+        fluxSinkFactory,
+        decoder,
+        fluxSource,
+        std::ranges::views::keys(diskLayout.layoutByLogicalLocation) |
+            std::ranges::to<std::vector>());
 }
 
-void writeRawDiskCommand(FluxSource& fluxSource, FluxSink& fluxSink)
+void writeRawDiskCommand(const DiskLayout& diskLayout,
+    FluxSource& fluxSource,
+    FluxSinkFactory& fluxSinkFactory)
 {
-    auto physicalLocations = Layout::computePhysicalLocations();
-    auto trackinfos = Layout::getLayoutOfTracksPhysical(physicalLocations);
     writeTracks(
-        fluxSink,
+        diskLayout,
-        [&](std::shared_ptr<const TrackInfo>& trackInfo)
+        fluxSinkFactory,
+        [&](const std::shared_ptr<const LogicalTrackLayout>& ltl)
         {
             return fluxSource
-                .readFlux(trackInfo->physicalTrack, trackInfo->physicalSide)
+                .readFlux(ltl->physicalCylinder, ltl->physicalHead)
                 ->next();
         },
         [](const auto&)
         {
             return true;
         },
-        trackinfos);
+        diskLayout.logicalLocations);
 }
 
-std::shared_ptr<TrackFlux> readAndDecodeTrack(FluxSource& fluxSource,
+void readAndDecodeTrack(const DiskLayout& diskLayout,
+    FluxSource& fluxSource,
     Decoder& decoder,
-    std::shared_ptr<const TrackInfo>& trackInfo)
+    const std::shared_ptr<const LogicalTrackLayout>& ltl,
+    std::vector<std::shared_ptr<const Track>>& tracks,
+    std::vector<std::shared_ptr<const Sector>>& combinedSectors)
 {
-    auto trackFlux = std::make_shared<TrackFlux>();
-    trackFlux->trackInfo = trackInfo;
-
     if (fluxSource.isHardware())
         measureDiskRotation();
 
@@ -595,8 +669,9 @@ std::shared_ptr<TrackFlux> readAndDecodeTrack(FluxSource& fluxSource,
     int retriesRemaining = globalConfig()->decoder().retries();
     for (;;)
     {
-        auto result =
+        auto [result, sectors] = readGroup(
-            readGroup(fluxSourceIteratorHolder, trackInfo, *trackFlux, decoder);
+            diskLayout, fluxSourceIteratorHolder, ltl, tracks, decoder);
+        combinedSectors = sectors;
         if (result == GOOD_READ)
             break;
         if (result == BAD_AND_CAN_NOT_RETRY)
@@ -613,131 +688,172 @@ std::shared_ptr<TrackFlux> readAndDecodeTrack(FluxSource& fluxSource,
 
         if (fluxSource.isHardware())
         {
-            adjustTrackOnError(fluxSource, trackInfo->physicalTrack);
+            adjustTrackOnError(fluxSource, ltl->physicalCylinder);
             log("retrying; {} retries remaining", retriesRemaining);
             retriesRemaining--;
         }
     }
-
-    return trackFlux;
 }
 
-std::shared_ptr<const DiskFlux> readDiskCommand(
+void readDiskCommand(const DiskLayout& diskLayout,
-    FluxSource& fluxSource, Decoder& decoder)
+    FluxSource& fluxSource,
+    Decoder& decoder,
+    Disk& disk)
 {
-    std::unique_ptr<FluxSink> outputFluxSink;
+    std::unique_ptr<FluxSinkFactory> outputFluxSinkFactory;
     if (globalConfig()->decoder().has_copy_flux_to())
-        outputFluxSink =
+        outputFluxSinkFactory =
-            FluxSink::create(globalConfig()->decoder().copy_flux_to());
+            FluxSinkFactory::create(globalConfig()->decoder().copy_flux_to());
 
-    auto diskflux = std::make_shared<DiskFlux>();
+    std::map<CylinderHead, std::vector<std::shared_ptr<const Track>>>
+        tracksByLogicalLocation;
+    for (auto& [ch, track] : disk.tracksByPhysicalLocation)
+        tracksByLogicalLocation[CylinderHead(track->ltl->logicalCylinder,
+                                    track->ltl->logicalHead)]
+            .push_back(track);
 
     log(BeginOperationLogMessage{"Reading and decoding disk"});
-    auto physicalLocations = Layout::computePhysicalLocations();
+
-    unsigned index = 0;
+    if (fluxSource.isHardware())
-    for (auto& physicalLocation : physicalLocations)
+        disk.rotationalPeriod = measureDiskRotation();
+    else
+        disk.rotationalPeriod = getRotationalPeriodFromConfig();
+
     {
-        auto trackInfo = Layout::getLayoutOfTrackPhysical(
+        std::unique_ptr<FluxSink> outputFluxSink;
-            physicalLocation.cylinder, physicalLocation.head);
+        if (outputFluxSinkFactory)
-
+            outputFluxSink = outputFluxSinkFactory->create();
-        log(OperationProgressLogMessage{
+        unsigned index = 0;
-            index * 100 / (unsigned)physicalLocations.size()});
+        for (auto& [logicalLocation, ltl] : diskLayout.layoutByLogicalLocation)
-        index++;
-
-        testForEmergencyStop();
-
-        auto trackFlux = readAndDecodeTrack(fluxSource, decoder, trackInfo);
-        diskflux->tracks.push_back(trackFlux);
-
-        if (outputFluxSink)
         {
-            for (const auto& data : trackFlux->trackDatas)
+            log(OperationProgressLogMessage{
-                outputFluxSink->writeFlux(trackInfo->physicalTrack,
+                index * 100 /
-                    trackInfo->physicalSide,
+                (unsigned)diskLayout.layoutByLogicalLocation.size()});
-                    *data->fluxmap);
+            index++;
-        }
 
-        if (globalConfig()->decoder().dump_records())
+            testForEmergencyStop();
-        {
-            std::vector<std::shared_ptr<const Record>> sorted_records;
 
-            for (const auto& data : trackFlux->trackDatas)
+            auto& trackFluxes = tracksByLogicalLocation[logicalLocation];
-                sorted_records.insert(sorted_records.end(),
+            std::vector<std::shared_ptr<const Sector>> trackSectors;
-                    data->records.begin(),
+            readAndDecodeTrack(diskLayout,
-                    data->records.end());
+                fluxSource,
+                decoder,
+                ltl,
+                trackFluxes,
+                trackSectors);
 
-            std::sort(sorted_records.begin(),
+            /* Replace all tracks on the disk by the new combined set. */
-                sorted_records.end(),
-                [](const auto& o1, const auto& o2)
-                {
-                    return o1->startTime < o2->startTime;
-                });
 
-            std::cout << "\nRaw (undecoded) records follow:\n\n";
+            for (const auto& flux : trackFluxes)
-            for (const auto& record : sorted_records)
+                disk.tracksByPhysicalLocation.erase(CylinderHead{
+                    flux->ptl->physicalCylinder, flux->ptl->physicalHead});
+            for (const auto& flux : trackFluxes)
+                disk.tracksByPhysicalLocation.emplace(
+                    CylinderHead{
+                        flux->ptl->physicalCylinder, flux->ptl->physicalHead},
+                    flux);
+
+            /* Likewise for sectors. */
+
+            for (const auto& sector : trackSectors)
+                disk.sectorsByPhysicalLocation.erase(
+                    sector->physicalLocation.value());
+            for (const auto& sector : trackSectors)
+                disk.sectorsByPhysicalLocation.emplace(
+                    sector->physicalLocation.value(), sector);
+
+            if (outputFluxSink)
             {
-                std::cout << fmt::format("I+{:.2f}us with {:.2f}us clock\n",
+                for (const auto& data : trackFluxes)
-                    record->startTime / 1000.0,
+                    outputFluxSink->addFlux(data->ptl->physicalCylinder,
-                    record->clock / 1000.0);
+                        data->ptl->physicalHead,
-                hexdump(std::cout, record->rawData);
+                        *data->fluxmap);
-                std::cout << std::endl;
             }
-        }
 
-        if (globalConfig()->decoder().dump_sectors())
+            if (globalConfig()->decoder().dump_records())
-        {
-            auto collected_sectors = collectSectors(trackFlux->sectors, false);
-            std::vector<std::shared_ptr<const Sector>> sorted_sectors(
-                collected_sectors.begin(), collected_sectors.end());
-            std::sort(sorted_sectors.begin(),
-                sorted_sectors.end(),
-                [](const auto& o1, const auto& o2)
-                {
-                    return *o1 < *o2;
-                });
-
-            std::cout << "\nDecoded sectors follow:\n\n";
-            for (const auto& sector : sorted_sectors)
             {
-                std::cout << fmt::format(
+                std::vector<std::shared_ptr<const Record>> sorted_records;
-                    "{}.{:02}.{:02}: I+{:.2f}us with {:.2f}us clock: "
-                    "status {}\n",
-                    sector->logicalTrack,
-                    sector->logicalSide,
-                    sector->logicalSector,
-                    sector->headerStartTime / 1000.0,
-                    sector->clock / 1000.0,
-                    Sector::statusToString(sector->status));
-                hexdump(std::cout, sector->data);
-                std::cout << std::endl;
-            }
-        }
 
-        /* track can't be modified below this point. */
+                for (const auto& data : trackFluxes)
-        log(TrackReadLogMessage{trackFlux});
+                    sorted_records.insert(sorted_records.end(),
+                        data->records.begin(),
+                        data->records.end());
+
+                std::sort(sorted_records.begin(),
+                    sorted_records.end(),
+                    [](const auto& o1, const auto& o2)
+                    {
+                        return o1->startTime < o2->startTime;
+                    });
+
+                std::cout << "\nRaw (undecoded) records follow:\n\n";
+                for (const auto& record : sorted_records)
+                {
+                    std::cout << fmt::format("I+{:.2f}us with {:.2f}us clock\n",
+                        record->startTime / 1000.0,
+                        record->clock / 1000.0);
+                    hexdump(std::cout, record->rawData);
+                    std::cout << std::endl;
+                }
+            }
+
+            if (globalConfig()->decoder().dump_sectors())
+            {
+                auto sectors = collectSectors(trackSectors, false);
+                std::ranges::sort(sectors,
+                    [](const auto& o1, const auto& o2)
+                    {
+                        return *o1 < *o2;
+                    });
+
+                std::cout << "\nDecoded sectors follow:\n\n";
+                for (const auto& sector : sectors)
+                {
+                    std::cout << fmt::format(
+                        "{}.{:02}.{:02}: I+{:.2f}us with {:.2f}us clock: "
+                        "status {}\n",
+                        sector->logicalCylinder,
+                        sector->logicalHead,
+                        sector->logicalSector,
+                        sector->headerStartTime / 1000.0,
+                        sector->clock / 1000.0,
+                        Sector::statusToString(sector->status));
+                    hexdump(std::cout, sector->data);
+                    std::cout << std::endl;
+                }
+            }
+
+            /* track can't be modified below this point. */
+            log(TrackReadLogMessage{trackFluxes, trackSectors});
+
+            std::vector<std::shared_ptr<const Sector>> all_sectors;
+            for (auto& [ch, sector] : disk.sectorsByPhysicalLocation)
+                all_sectors.push_back(sector);
+            all_sectors = collectSectors(all_sectors);
+            disk.image = std::make_shared<Image>(all_sectors);
+
+            /* Log a _copy_ of the disk structure so that the logger
+             * doesn't see the disk get mutated in subsequent reads. */
+            log(DiskReadLogMessage{std::make_shared<Disk>(disk)});
+        }
     }
 
-    std::set<std::shared_ptr<const Sector>> all_sectors;
+    if (!disk.image)
-    for (auto& track : diskflux->tracks)
+        disk.image = std::make_shared<Image>();
-        for (auto& sector : track->sectors)
-            all_sectors.insert(sector);
-    all_sectors = collectSectors(all_sectors);
-    diskflux->image = std::make_shared<Image>(all_sectors);
 
-    /* diskflux can't be modified below this point. */
-    log(DiskReadLogMessage{diskflux});
     log(EndOperationLogMessage{"Read complete"});
-    return diskflux;
 }
 
-void readDiskCommand(
+void readDiskCommand(const DiskLayout& diskLayout,
-    FluxSource& fluxsource, Decoder& decoder, ImageWriter& writer)
+    FluxSource& fluxSource,
+    Decoder& decoder,
+    ImageWriter& writer)
 {
-    auto diskflux = readDiskCommand(fluxsource, decoder);
+    Disk disk;
+    readDiskCommand(diskLayout, fluxSource, decoder, disk);
 
-    writer.printMap(*diskflux->image);
+    writer.printMap(*disk.image);
     if (globalConfig()->decoder().has_write_csv_to())
-        writer.writeCsv(
+        writer.writeCsv(*disk.image, globalConfig()->decoder().write_csv_to());
-            *diskflux->image, globalConfig()->decoder().write_csv_to());
+    writer.writeImage(*disk.image);
-    writer.writeImage(*diskflux->image);
 }

lib/algorithms/readerwriter.h

@@ -3,19 +3,20 @@
 
 #include "lib/data/locations.h"
 
+class Disk;
+class Track;
 class Decoder;
+class DiskLayout;
 class Encoder;
-class DiskFlux;
+class FluxSinkFactory;
-class FluxSink;
 class FluxSource;
 class FluxSourceIteratorHolder;
 class Fluxmap;
 class Image;
 class ImageReader;
 class ImageWriter;
-class TrackInfo;
+class LogicalTrackLayout;
-class TrackFlux;
+class PhysicalTrackLayout;
-class TrackDataFlux;
 class Sector;
 
 struct BeginSpeedOperationLogMessage
@@ -29,12 +30,13 @@ struct EndSpeedOperationLogMessage
 
 struct TrackReadLogMessage
 {
-    std::shared_ptr<const TrackFlux> track;
+    std::vector<std::shared_ptr<const Track>> tracks;
+    std::vector<std::shared_ptr<const Sector>> sectors;
 };
 
 struct DiskReadLogMessage
 {
-    std::shared_ptr<const DiskFlux> disk;
+    std::shared_ptr<const Disk> disk;
 };
 
 struct BeginReadOperationLogMessage
@@ -45,7 +47,7 @@ struct BeginReadOperationLogMessage
 
 struct EndReadOperationLogMessage
 {
-    std::shared_ptr<const TrackDataFlux> trackDataFlux;
+    std::shared_ptr<const Track> trackDataFlux;
     std::set<std::shared_ptr<const Sector>> sectors;
 };
 
@@ -74,42 +76,56 @@ struct OperationProgressLogMessage
     unsigned progress;
 };
 
-extern void measureDiskRotation();
+extern void writeTracks(const DiskLayout& diskLayout,
-
+    FluxSinkFactory& fluxSinkFactory,
-extern void writeTracks(FluxSink& fluxSink,
     const std::function<std::unique_ptr<const Fluxmap>(
-        std::shared_ptr<const TrackInfo>& layout)> producer,
+        const LogicalTrackLayout& ltl)> producer,
-    std::vector<std::shared_ptr<const TrackInfo>>& locations);
+    const std::vector<CylinderHead>& locations);
 
-extern void writeTracksAndVerify(FluxSink& fluxSink,
+extern void writeTracksAndVerify(const DiskLayout& diskLayout,
+    FluxSinkFactory& fluxSinkFactory,
     Encoder& encoder,
     FluxSource& fluxSource,
     Decoder& decoder,
     const Image& image,
-    std::vector<std::shared_ptr<const TrackInfo>>& locations);
+    const std::vector<CylinderHead>& locations);
 
-extern void writeDiskCommand(const Image& image,
+extern void writeDiskCommand(const DiskLayout& diskLayout,
+    const Image& image,
     Encoder& encoder,
-    FluxSink& fluxSink,
+    FluxSinkFactory& fluxSinkFactory,
     Decoder* decoder,
     FluxSource* fluxSource,
     const std::vector<CylinderHead>& locations);
 
-extern void writeDiskCommand(const Image& image,
+extern void writeDiskCommand(const DiskLayout& diskLayout,
+    const Image& image,
     Encoder& encoder,
-    FluxSink& fluxSink,
+    FluxSinkFactory& fluxSinkFactory,
     Decoder* decoder = nullptr,
     FluxSource* fluxSource = nullptr);
 
-extern void writeRawDiskCommand(FluxSource& fluxSource, FluxSink& fluxSink);
+extern void writeRawDiskCommand(const DiskLayout& diskLayout,
+    FluxSource& fluxSource,
+    FluxSinkFactory& fluxSinkFactory);
 
-extern std::shared_ptr<TrackFlux> readAndDecodeTrack(FluxSource& fluxSource,
+/* Reads a single group of tracks. tracks and combinedSectors are populated.
+ * tracks may contain preexisting data which will be taken into account. */
+
+extern void readAndDecodeTrack(const DiskLayout& diskLayout,
+    FluxSource& fluxSource,
     Decoder& decoder,
-    std::shared_ptr<const TrackInfo>& layout);
+    const std::shared_ptr<const LogicalTrackLayout>& ltl,
+    std::vector<std::shared_ptr<const Track>>& tracks,
+    std::vector<std::shared_ptr<const Sector>>& combinedSectors);
 
-extern std::shared_ptr<const DiskFlux> readDiskCommand(
+extern void readDiskCommand(const DiskLayout& diskLayout,
-    FluxSource& fluxsource, Decoder& decoder);
+    FluxSource& fluxSource,
-extern void readDiskCommand(
+    Decoder& decoder,
-    FluxSource& source, Decoder& decoder, ImageWriter& writer);
+    Disk& disk);
+extern void readDiskCommand(const DiskLayout& diskLayout,
+    FluxSource& source,
+    Decoder& decoder,
+    ImageWriter& writer);
 
 #endif

lib/config/config.cc

@@ -211,6 +211,13 @@ void Config::clear()
     _appliedOptions.clear();
 }
 
+static std::string getValidValues(const OptionGroupProto& group)
+{
+    return fmt::format("{}",
+        fmt::join(
+            std::views::transform(group.option(), &OptionProto::name), ", "));
+}
+
 std::vector<std::string> Config::validate()
 {
     std::vector<std::string> results;
@@ -218,7 +225,7 @@ std::vector<std::string> Config::validate()
     /* Ensure that only one item in each group is set. */
 
     std::map<const OptionGroupProto*, const OptionProto*> optionsByGroup;
-    for (auto [group, option, hasArgument] : _appliedOptions)
+    for (auto& [group, option, hasArgument] : _appliedOptions)
         if (group)
         {
             auto& o = optionsByGroup[group];
@@ -227,12 +234,23 @@ std::vector<std::string> Config::validate()
                     fmt::format("multiple mutually exclusive values set for "
                                 "group '{}': valid values are: {}",
                         group->comment(),
-                        fmt::join(std::views::transform(
+                        getValidValues(*group)));
-                                      group->option(), &OptionProto::name),
-                            ", ")));
             o = option;
         }
 
+    /* Ensure that every group has an option set. */
+
+    for (const auto& group : base()->option_group())
+    {
+        if (!optionsByGroup.contains(&group))
+        {
+            results.push_back(
+                fmt::format("no value set for group '{}': valid values are: {}",
+                    group.comment(),
+                    getValidValues(group)));
+        }
+    }
+
     /* Check option requirements. */
 
     for (auto [group, option, hasArgument] : _appliedOptions)
@@ -357,7 +375,7 @@ Config::OptionInfo Config::findOption(
     {
         if (optionGroup.name().empty())
             if (searchOptionList(optionGroup.option(), name))
-                return {nullptr, found, false};
+                return {&optionGroup, found, false};
     }
 
     throw OptionNotFoundException(fmt::format("option {} not found", name));
@@ -395,8 +413,7 @@ void Config::checkOptionValid(const OptionProto& option)
             ss << ']';
 
             throw InapplicableOptionException(
-                fmt::format("option '{}' is inapplicable to this "
+                fmt::format("option '{}' is inapplicable to this configuration "
-                            "configuration "
                             "because {}={} could not be met",
                     option.name(),
                     req.key(),
@@ -434,6 +451,52 @@ bool Config::applyOption(const std::string& name, const std::string value)
     return optionInfo.usesValue;
 }
 
+void Config::applyOptionsFile(const std::string& data)
+{
+    if (!data.empty())
+    {
+        for (auto setting : split(data, '\n'))
+        {
+            setting = trimWhitespace(setting);
+            if (setting.size() == 0)
+                continue;
+            if (setting[0] == '#')
+                continue;
+
+            auto equals = setting.find('=');
+            if (equals == std::string::npos)
+                error("Malformed setting line '{}'", setting);
+
+            auto key = setting.substr(0, equals);
+            auto value = setting.substr(equals + 1);
+            globalConfig().set(key, value);
+        }
+    }
+}
+
+void Config::applyDefaultOptions()
+{
+    std::set<const OptionGroupProto*> appliedOptionGroups;
+    for (auto& [group, option, hasArgument] : _appliedOptions)
+        if (group)
+            appliedOptionGroups.insert(group);
+
+    /* For every group which doesn't have an option set, find the default and
+     * set it. */
+
+    for (const auto& group : base()->option_group())
+    {
+        if (!appliedOptionGroups.contains(&group))
+        {
+            for (const auto& option : group.option())
+            {
+                if (option.set_by_default())
+                    applyOption({&group, &option, false});
+            }
+        }
+    }
+}
+
 void Config::clearOptions()
 {
     _appliedOptions.clear();

lib/config/config.h

@@ -9,7 +9,7 @@
 class ConfigProto;
 class OptionProto;
 class FluxSource;
-class FluxSink;
+class FluxSinkFactory;
 class ImageReader;
 class ImageWriter;
 class Encoder;
@@ -142,6 +142,8 @@ public:
     bool isOptionValid(const OptionProto& option);
     void applyOption(const OptionInfo& optionInfo);
     bool applyOption(const std::string& name, const std::string value = "");
+    void applyOptionsFile(const std::string& data);
+    void applyDefaultOptions();
     void clearOptions();
 
     /* Adjust overall inputs and outputs. */

lib/config/config.proto

@@ -55,7 +55,16 @@ message OptionPrerequisiteProto
     repeated string value = 2 [(help) = "list of required values"];
 }
 
-// NEXT_TAG: 8
+enum OptionApplicabilityHint
+{
+    FORMAT = 0;
+    ANY_SOURCESINK = 1;
+    HARDWARE_SOURCESINK = 2;
+	MANUAL_SOURCESINK = 3;
+	FLUXFILE_SOURCESINK = 4;
+}
+
+// NEXT_TAG: 9
 message OptionProto
 {
     optional string name = 1 [(help) = "option name"];
@@ -68,11 +77,14 @@ message OptionProto
         7 [(help) = "prerequisites for this option"];
 
     optional ConfigProto config = 4 [(help) = "option data"];
+    repeated OptionApplicabilityHint applicability = 8;
 }
 
+// NEXT_TAG: 5
 message OptionGroupProto
 {
     optional string comment = 1 [(help) = "help text for option group"];
     optional string name = 2 [(help) = "option group name"];
     repeated OptionProto option = 3;
+    repeated OptionApplicabilityHint applicability = 4;
 }

lib/config/drive.proto

@@ -16,11 +16,11 @@ message DriveProto
         [ default = 0, (help) = "index pulses longer than this interval are "
             "considered sector markers; shorter indicates an true index marker" ];
     optional bool high_density = 5
-        [ default = true, (help) = "set if this is a high density disk" ];
+        [ default = false, (help) = "set if this is a high density disk" ];
     optional bool sync_with_index = 6
         [ default = false, (help) = "start reading at index mark" ];
     optional double revolutions = 7
-        [ default = 1.2, (help) = "number of revolutions to read" ];
+        [ default = 2.5, (help) = "number of revolutions to read" ];
 
     optional string tracks = 8
         [ default = "c0-80h0-1", (help) = "Tracks supported by drive" ];

lib/config/flags.cc

@@ -174,6 +174,7 @@ std::vector<std::string> FlagGroup::parseFlagsWithFilenames(int argc,
         index++;
     }
 
+    globalConfig().applyDefaultOptions();
     globalConfig().validateAndThrow();
     return filenames;
 }

lib/config/proto.cc

@@ -117,38 +117,28 @@ static ProtoField resolveProtoPath(
                 "config field '{}' in '{}' is not a message", item, path));
 
         const auto* reflection = message->GetReflection();
-        if ((field->label() !=
+        if (!field->is_repeated() && (index != -1))
-                google::protobuf::FieldDescriptor::LABEL_REPEATED) &&
-            (index != -1))
             throw ProtoPathNotFoundException(fmt::format(
                 "config field '{}[{}]' is indexed, but not repeated",
                 item,
                 index));
 
-        switch (field->label())
+        if (field->is_repeated())
         {
-            case google::protobuf::FieldDescriptor::LABEL_OPTIONAL:
+            if (index == -1)
-            case google::protobuf::FieldDescriptor::LABEL_REQUIRED:
+                throw ProtoPathNotFoundException(fmt::format(
-                if (!create && !reflection->HasField(*message, field))
+                    "config field '{}' is repeated and must be indexed", item));
-                    throw ProtoPathNotFoundException(fmt::format(
+            while (reflection->FieldSize(*message, field) <= index)
-                        "could not find config field '{}'", field->name()));
+                reflection->AddMessage(message, field);
-                message = reflection->MutableMessage(message, field);
-                break;
 
-            case google::protobuf::FieldDescriptor::LABEL_REPEATED:
+            message = reflection->MutableRepeatedMessage(message, field, index);
-                if (index == -1)
+        }
-                    throw ProtoPathNotFoundException(fmt::format(
+        else
-                        "config field '{}' is repeated and must be indexed",
+        {
-                        item));
+            if (!create && !reflection->HasField(*message, field))
-                while (reflection->FieldSize(*message, field) <= index)
+                throw ProtoPathNotFoundException(fmt::format(
-                    reflection->AddMessage(message, field);
+                    "could not find config field '{}'", field->name()));
-
+            message = reflection->MutableMessage(message, field);
-                message =
-                    reflection->MutableRepeatedMessage(message, field, index);
-                break;
-
-            default:
-                error("bad proto label for field '{}' in '{}'", item, path);
         }
 
         descriptor = message->GetDescriptor();
@@ -215,7 +205,7 @@ static void updateRepeatedField(
 void ProtoField::set(const std::string& value)
 {
     const auto* reflection = _message->GetReflection();
-    if (_field->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+    if (_field->is_repeated())
     {
         if (_index == -1)
             error("field '{}' is repeated but no index is provided");
@@ -359,7 +349,7 @@ void ProtoField::set(const std::string& value)
 std::string ProtoField::get() const
 {
     const auto* reflection = _message->GetReflection();
-    if (_field->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+    if (_field->is_repeated())
     {
         if (_index == -1)
             error("field '{}' is repeated but no index is provided",
@@ -456,7 +446,7 @@ std::string ProtoField::get() const
 google::protobuf::Message* ProtoField::getMessage() const
 {
     const auto* reflection = _message->GetReflection();
-    if (_field->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+    if (_field->is_repeated())
     {
         if (_index == -1)
             error("field '{}' is repeated but no index is provided",
@@ -477,7 +467,7 @@ google::protobuf::Message* ProtoField::getMessage() const
 std::string ProtoField::getBytes() const
 {
     const auto* reflection = _message->GetReflection();
-    if (_field->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+    if (_field->is_repeated())
     {
         if (_index == -1)
             error("field '{}' is repeated but no index is provided",
@@ -536,7 +526,7 @@ findAllPossibleProtoFields(const google::protobuf::Descriptor* descriptor)
             const google::protobuf::FieldDescriptor* f = d->field(i);
             std::string n = s + (std::string)f->name();
 
-            if (f->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+            if (f->is_repeated())
                 n += "[]";
 
             if (shouldRecurse(f))
@@ -550,12 +540,13 @@ findAllPossibleProtoFields(const google::protobuf::Descriptor* descriptor)
     return fields;
 }
 
-std::vector<ProtoField> findAllProtoFields(google::protobuf::Message* message)
+std::vector<ProtoField> findAllProtoFields(
+    const google::protobuf::Message* message)
 {
     std::vector<ProtoField> allFields;
 
-    std::function<void(google::protobuf::Message*, const std::string&)>
+    std::function<void(const google::protobuf::Message*, const std::string&)>
-        recurse = [&](auto* message, const auto& name)
+        recurse = [&](const auto* message, const auto& name)
     {
         const auto* reflection = message->GetReflection();
         std::vector<const google::protobuf::FieldDescriptor*> fields;
@@ -568,25 +559,26 @@ std::vector<ProtoField> findAllProtoFields(google::protobuf::Message* message)
                 basename += '.';
             basename += f->name();
 
-            if (f->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+            if (f->is_repeated())
             {
                 for (int i = 0; i < reflection->FieldSize(*message, f); i++)
                 {
                     const auto n = fmt::format("{}[{}]", basename, i);
                     if (shouldRecurse(f))
                         recurse(
-                            reflection->MutableRepeatedMessage(message, f, i),
+                            &reflection->GetRepeatedMessage(*message, f, i), n);
-                            n);
                     else
-                        allFields.push_back(ProtoField(n, message, f, i));
+                        allFields.push_back(ProtoField(
+                            n, (google::protobuf::Message*)message, f, i));
                 }
             }
             else
             {
                 if (shouldRecurse(f))
-                    recurse(reflection->MutableMessage(message, f), basename);
+                    recurse(&reflection->GetMessage(*message, f), basename);
                 else
-                    allFields.push_back(ProtoField(basename, message, f));
+                    allFields.push_back(ProtoField(
+                        basename, (google::protobuf::Message*)message, f));
             }
         }
     };
@@ -594,3 +586,12 @@ std::vector<ProtoField> findAllProtoFields(google::protobuf::Message* message)
     recurse(message, "");
     return allFields;
 }
+
+std::string renderProtoAsConfig(const google::protobuf::Message* message)
+{
+    auto allFields = findAllProtoFields(message);
+    std::stringstream ss;
+    for (const auto& field : allFields)
+        ss << field.path() << "=" << field.get() << "\n";
+    return ss.str();
+}

lib/config/proto.h

@@ -72,7 +72,9 @@ extern std::map<std::string, const google::protobuf::FieldDescriptor*>
 findAllPossibleProtoFields(const google::protobuf::Descriptor* descriptor);
 
 extern std::vector<ProtoField> findAllProtoFields(
-    google::protobuf::Message* message);
+    const google::protobuf::Message* message);
+extern std::string renderProtoAsConfig(
+    const google::protobuf::Message* message);
 
 template <class T>
 static inline const T parseProtoBytes(const std::string_view& bytes)

lib/core/build.py

@@ -23,5 +23,6 @@ cxxlibrary(
         "dep/agg",
         "dep/stb",
         "+fmt_lib",
+        "+z_lib",
     ],
 )

lib/core/globals.h

@@ -27,28 +27,27 @@
 #define STRINGIFY(a) XSTRINGIFY(a)
 #define XSTRINGIFY(a) #a
 
-template <class T>
-static inline std::vector<T> vector_of(T item)
-{
-    return std::vector<T>{item};
-}
-
 typedef double nanoseconds_t;
 class Bytes;
 
 extern double getCurrentTime();
 extern void hexdump(std::ostream& stream, const Bytes& bytes);
-extern void hexdumpForSrp16(std::ostream& stream, const Bytes& bytes);
 
-struct ErrorException
+struct ErrorException : public std::exception
 {
     ErrorException(const std::string& message): message(message) {}
 
     const std::string message;
 
+    const char* what() const throw() override;
     void print() const;
 };
 
+struct OutOfRangeException : public ErrorException
+{
+    OutOfRangeException(const std::string& message): ErrorException(message) {}
+};
+
 template <typename... Args>
 [[noreturn]] inline void error(fmt::string_view fstr, const Args&... args)
 {
@@ -71,4 +70,24 @@ struct overloaded : Ts...
 template <class... Ts>
 overloaded(Ts...) -> overloaded<Ts...>;
 
+template <typename K, typename V>
+inline const V& findOrDefault(
+    const std::map<K, V>& map, const K& key, const V& defaultValue = V())
+{
+    auto it = map.find(key);
+    if (it == map.end())
+        return defaultValue;
+    return it->second;
+}
+
+template <typename K, typename V>
+inline const std::optional<V> findOptionally(
+    const std::map<K, V>& map, const K& key, const V& defaultValue = V())
+{
+    auto it = map.find(key);
+    if (it == map.end())
+        return std::nullopt;
+    return std::make_optional(it->second);
+}
+
 #endif

lib/core/hexdump.cc

@@ -32,10 +32,3 @@ void hexdump(std::ostream& stream, const Bytes& buffer)
         pos += 16;
     }
 }
-
-void hexdumpForSrp16(std::ostream& stream, const Bytes& buffer)
-{
-    for (uint8_t byte : buffer)
-        stream << fmt::format("{:02x}", byte);
-    stream << std::endl;
-}

lib/core/logger.h

@@ -4,9 +4,8 @@
 #include "fmt/format.h"
 #include <type_traits>
 
-class DiskFlux;
+class Disk;
-class TrackDataFlux;
+class Track;
-class TrackFlux;
 class Sector;
 class LogRenderer;
 

lib/core/utils.cc

@@ -17,6 +17,11 @@ void ErrorException::print() const
     std::cerr << message << '\n';
 }
 
+const char* ErrorException::what() const throw()
+{
+    return message.c_str();
+}
+
 std::string join(
     const std::vector<std::string>& values, const std::string& separator)
 {

lib/core/utils.h

@@ -37,6 +37,15 @@ std::map<V, K> reverseMap(const std::map<K, V>& map)
     return reverse;
 }
 
+template <typename K, typename V>
+std::map<K, std::vector<V>> multimapToMapOfVectors(const std::multimap<K, V>& multimap)
+{
+    std::map<K, std::vector<V>> results;
+    for (auto& [k, v] : multimap)
+        results[k].push_back(v);
+    return results;
+}
+
 /* If set, any running job will terminate as soon as possible (with an error).
  */
 

lib/data/build.py

@@ -3,6 +3,7 @@ from build.c import cxxlibrary
 cxxlibrary(
     name="data",
     srcs=[
+        "./disk.cc",
         "./fluxmap.cc",
         "./fluxmapreader.cc",
         "./fluxpattern.cc",
@@ -12,7 +13,7 @@ cxxlibrary(
         "./sector.cc",
     ],
     hdrs={
-        "lib/data/flux.h": "./flux.h",
+        "lib/data/disk.h": "./disk.h",
         "lib/data/fluxmap.h": "./fluxmap.h",
         "lib/data/sector.h": "./sector.h",
         "lib/data/layout.h": "./layout.h",

lib/data/disk.cc

@@ -0,0 +1,56 @@
+#include "lib/core/globals.h"
+#include "lib/data/disk.h"
+#include "lib/data/image.h"
+#include "lib/data/layout.h"
+#include "lib/data/sector.h"
+#include "protocol.h"
+
+namespace
+{
+    struct pair_to_range_t
+    {
+        template <typename I>
+        friend constexpr auto operator|(
+            std::pair<I, I> const& pr, pair_to_range_t)
+        {
+            return std::ranges::subrange(pr.first, pr.second);
+        }
+    };
+    inline constexpr pair_to_range_t pair_to_range{};
+}
+
+Disk::Disk() {}
+
+Disk::Disk(
+    const std::shared_ptr<const Image>& image, const DiskLayout& diskLayout):
+    image(image)
+{
+    std::multimap<CylinderHead, std::shared_ptr<const Sector>>
+        sectorsGroupedByTrack;
+    for (const auto& sector : *image)
+        sectorsGroupedByTrack.insert(
+            std::make_pair(sector->physicalLocation.value(), sector));
+
+    for (const auto& physicalLocation :
+        std::views::keys(sectorsGroupedByTrack) | std::ranges::to<std::set>())
+    {
+        const auto& ptl =
+            diskLayout.layoutByPhysicalLocation.at(physicalLocation);
+        const auto& ltl = ptl->logicalTrackLayout;
+
+        auto decodedTrack = std::make_shared<Track>();
+        decodedTrack->ltl = ltl;
+        decodedTrack->ptl = ptl;
+        tracksByPhysicalLocation.insert(
+            std::make_pair(physicalLocation, decodedTrack));
+
+        for (auto& [ch, sector] :
+            sectorsGroupedByTrack.equal_range(physicalLocation) | pair_to_range)
+        {
+            decodedTrack->allSectors.push_back(sector);
+            decodedTrack->normalisedSectors.push_back(sector);
+            sectorsByPhysicalLocation.insert(
+                std::make_pair(physicalLocation, sector));
+        }
+    }
+}

lib/data/disk.h

@@ -0,0 +1,62 @@
+#ifndef FLUX_H
+#define FLUX_H
+
+#include "lib/core/bytes.h"
+#include "lib/data/locations.h"
+
+class DiskLayout;
+class Fluxmap;
+class Image;
+class LogicalTrackLayout;
+class PhysicalTrackLayout;
+class Sector;
+
+struct Record
+{
+    nanoseconds_t clock = 0;
+    nanoseconds_t startTime = 0;
+    nanoseconds_t endTime = 0;
+    uint32_t position = 0;
+    Bytes rawData;
+};
+
+struct Track
+{
+    std::shared_ptr<const LogicalTrackLayout> ltl;
+    std::shared_ptr<const PhysicalTrackLayout> ptl;
+    std::shared_ptr<const Fluxmap> fluxmap;
+    std::vector<std::shared_ptr<const Record>> records;
+    
+    /* All sectors, valid or not, including duplicates. */
+
+    std::vector<std::shared_ptr<const Sector>> allSectors;
+
+    /* Zero or one sector for each ID, preferring good ones. */
+
+    std::vector<std::shared_ptr<const Sector>> normalisedSectors;
+};
+
+struct Disk
+{
+    Disk();
+
+    /* Creates a Disk from an Image, populating the tracks and sectors maps
+     * based on the supplied disk layout. */
+
+    Disk(const std::shared_ptr<const Image>& image,
+        const DiskLayout& diskLayout);
+
+    Disk& operator=(const Disk& other) = default;
+
+    std::multimap<CylinderHead, std::shared_ptr<const Track>>
+        tracksByPhysicalLocation;
+    std::multimap<CylinderHead, std::shared_ptr<const Sector>>
+        sectorsByPhysicalLocation;
+    std::shared_ptr<const Image> image;
+
+    /* 0 if the period is unknown (e.g. if this Disk was made from an image). */
+
+    nanoseconds_t rotationalPeriod = 0;
+};
+
+#endif

lib/data/flux.h

@@ -1,41 +0,0 @@
-#ifndef FLUX_H
-#define FLUX_H
-
-#include "lib/core/bytes.h"
-
-class Fluxmap;
-class Sector;
-class Image;
-class TrackInfo;
-
-struct Record
-{
-    nanoseconds_t clock = 0;
-    nanoseconds_t startTime = 0;
-    nanoseconds_t endTime = 0;
-    uint32_t position = 0;
-    Bytes rawData;
-};
-
-struct TrackDataFlux
-{
-    std::shared_ptr<const TrackInfo> trackInfo;
-    std::shared_ptr<const Fluxmap> fluxmap;
-    std::vector<std::shared_ptr<const Record>> records;
-    std::vector<std::shared_ptr<const Sector>> sectors;
-};
-
-struct TrackFlux
-{
-    std::shared_ptr<const TrackInfo> trackInfo;
-    std::vector<std::shared_ptr<TrackDataFlux>> trackDatas;
-    std::set<std::shared_ptr<const Sector>> sectors;
-};
-
-struct DiskFlux
-{
-    std::vector<std::shared_ptr<TrackFlux>> tracks;
-    std::shared_ptr<const Image> image;
-};
-
-#endif

lib/data/fluxmap.cc

@@ -1,6 +1,8 @@
 #include "lib/core/globals.h"
 #include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
 #include "protocol.h"
+#include <mutex>
 
 Fluxmap& Fluxmap::appendBytes(const Bytes& bytes)
 {
@@ -12,6 +14,8 @@ Fluxmap& Fluxmap::appendBytes(const Bytes& bytes)
 
 Fluxmap& Fluxmap::appendBytes(const uint8_t* ptr, size_t len)
 {
+    flushIndexMarks();
+
     ByteWriter bw(_bytes);
     bw.seekToEnd();
 
@@ -52,6 +56,7 @@ Fluxmap& Fluxmap::appendPulse()
 
 Fluxmap& Fluxmap::appendIndex()
 {
+    flushIndexMarks();
     findLastByte() |= 0x40;
     return *this;
 }
@@ -75,3 +80,33 @@ std::vector<std::unique_ptr<const Fluxmap>> Fluxmap::split() const
 
     return maps;
 }
+
+const std::vector<nanoseconds_t>& Fluxmap::getIndexMarks() const
+{
+    std::scoped_lock lock(_mutationMutex);
+    if (!_indexMarks.has_value())
+    {
+        _indexMarks = std::make_optional<std::vector<nanoseconds_t>>();
+        FluxmapReader fmr(*this);
+        nanoseconds_t oldt = -1;
+        for (;;)
+        {
+            unsigned ticks;
+            if (!fmr.findEvent(F_BIT_INDEX, ticks))
+                break;
+
+            /* Debounce. */
+            nanoseconds_t t = fmr.tell().ns();
+            if (t != oldt)
+                _indexMarks->push_back(t);
+            oldt = t;
+        }
+    }
+    return *_indexMarks;
+}
+
+void Fluxmap::flushIndexMarks()
+{
+    std::scoped_lock lock(_mutationMutex);
+    _indexMarks = {};
+}

lib/data/fluxmap.h

@@ -4,6 +4,7 @@
 #include "lib/core/bytes.h"
 #include "protocol.h"
 #include "fmt/format.h"
+#include <mutex>
 
 class RawBits;
 
@@ -82,14 +83,18 @@ public:
     std::unique_ptr<const Fluxmap> precompensate(
         int threshold_ticks, int amount_ticks);
     std::vector<std::unique_ptr<const Fluxmap>> split() const;
+    const std::vector<nanoseconds_t>& getIndexMarks() const;
 
 private:
     uint8_t& findLastByte();
+    void flushIndexMarks();
 
 private:
     nanoseconds_t _duration = 0;
     int _ticks = 0;
     Bytes _bytes;
+    mutable std::mutex _mutationMutex;
+    mutable std::optional<std::vector<nanoseconds_t>> _indexMarks;
 };
 
 #endif

lib/data/fluxmapreader.cc

@@ -44,6 +44,13 @@ void FluxmapReader::skipToEvent(int event)
     findEvent(event, ticks);
 }
 
+int FluxmapReader::getCurrentEvent()
+{
+    if (eof())
+        return F_EOF;
+    return _bytes[_pos.bytes] & 0xc0;
+}
+
 bool FluxmapReader::findEvent(int event, unsigned& ticks)
 {
     ticks = 0;

lib/data/fluxmapreader.h

@@ -42,6 +42,7 @@ public:
         return (_fluxmap.duration());
     }
 
+    int getCurrentEvent();
     void getNextEvent(int& event, unsigned& ticks);
     void skipToEvent(int event);
     bool findEvent(int event, unsigned& ticks);

lib/data/image.cc

@@ -6,14 +6,13 @@
 
 Image::Image() {}
 
-Image::Image(std::set<std::shared_ptr<const Sector>>& sectors)
+Image::Image(const std::vector<std::shared_ptr<const Sector>>& sectors)
 {
     for (auto& sector : sectors)
-    {
+        _sectors[{sector->logicalCylinder,
-        key_t key = std::make_tuple(
+            sector->logicalHead,
-            sector->logicalTrack, sector->logicalSide, sector->logicalSector);
+            sector->logicalSector}] = sector;
-        _sectors[key] = sector;
+
-    }
     calculateSize();
 }
 
@@ -23,72 +22,53 @@ void Image::clear()
     _geometry = {0, 0, 0};
 }
 
-void Image::createBlankImage()
-{
-    clear();
-    for (const auto& trackAndHead : Layout::getTrackOrdering(
-             globalConfig()->layout().filesystem_track_order()))
-    {
-        unsigned track = trackAndHead.first;
-        unsigned side = trackAndHead.second;
-        auto trackLayout = Layout::getLayoutOfTrack(track, side);
-        Bytes blank(trackLayout->sectorSize);
-        for (unsigned sectorId : trackLayout->naturalSectorOrder)
-            put(track, side, sectorId)->data = blank;
-    }
-}
-
 bool Image::empty() const
 {
     return _sectors.empty();
 }
 
-bool Image::contains(unsigned track, unsigned side, unsigned sectorid) const
+bool Image::contains(const LogicalLocation& location) const
 {
-    key_t key = std::make_tuple(track, side, sectorid);
+    return _sectors.find(location) != _sectors.end();
-    return _sectors.find(key) != _sectors.end();
 }
 
-std::shared_ptr<const Sector> Image::get(
+std::shared_ptr<const Sector> Image::get(const LogicalLocation& location) const
-    unsigned track, unsigned side, unsigned sectorid) const
 {
-    static std::shared_ptr<const Sector> NONE;
+    auto i = _sectors.find(location);
-
-    key_t key = std::make_tuple(track, side, sectorid);
-    auto i = _sectors.find(key);
     if (i == _sectors.end())
-        return NONE;
+        return nullptr;
     return i->second;
 }
 
-std::shared_ptr<Sector> Image::put(
+std::shared_ptr<Sector> Image::put(const LogicalLocation& location)
-    unsigned track, unsigned side, unsigned sectorid)
 {
-    auto trackLayout = Layout::getLayoutOfTrack(track, side);
+    auto sector = std::make_shared<Sector>(location);
-    key_t key = std::make_tuple(track, side, sectorid);
+    _sectors[location] = sector;
-    std::shared_ptr<Sector> sector = std::make_shared<Sector>();
-    sector->logicalTrack = track;
-    sector->logicalSide = side;
-    sector->logicalSector = sectorid;
-    sector->physicalTrack = Layout::remapTrackLogicalToPhysical(track);
-    sector->physicalSide = side;
-    _sectors[key] = sector;
     return sector;
 }
 
-void Image::erase(unsigned track, unsigned side, unsigned sectorid)
+void Image::erase(const LogicalLocation& location)
 {
-    key_t key = std::make_tuple(track, side, sectorid);
+    _sectors.erase(location);
-    _sectors.erase(key);
 }
 
-std::set<std::pair<unsigned, unsigned>> Image::tracks() const
+void Image::addMissingSectors(const DiskLayout& diskLayout, bool populated)
 {
-    std::set<std::pair<unsigned, unsigned>> result;
+    for (auto& location : diskLayout.logicalSectorLocationsInFilesystemOrder)
-    for (const auto& e : _sectors)
+        if (!_sectors.contains(location))
-        result.insert(
+        {
-            std::make_pair(std::get<0>(e.first), std::get<1>(e.first)));
+            auto& ltl = diskLayout.layoutByLogicalLocation.at(
-    return result;
+                {location.logicalCylinder, location.logicalHead});
+            auto sector = std::make_shared<Sector>(location);
+
+            if (populated)
+                sector->data = Bytes(ltl->sectorSize);
+            else
+                sector->status = Sector::MISSING;
+
+            _sectors[location] = sector;
+        }
+    calculateSize();
 }
 
 void Image::calculateSize()
@@ -100,16 +80,39 @@ void Image::calculateSize()
         const auto& sector = i.second;
         if (sector)
         {
-            _geometry.numTracks = std::max(
+            _geometry.numCylinders = std::max(
-                _geometry.numTracks, (unsigned)sector->logicalTrack + 1);
+                _geometry.numCylinders, (unsigned)sector->logicalCylinder + 1);
-            _geometry.numSides =
+            _geometry.numHeads =
-                std::max(_geometry.numSides, (unsigned)sector->logicalSide + 1);
+                std::max(_geometry.numHeads, (unsigned)sector->logicalHead + 1);
             _geometry.firstSector = std::min(
                 _geometry.firstSector, (unsigned)sector->logicalSector);
             maxSector = std::max(maxSector, (unsigned)sector->logicalSector);
             _geometry.sectorSize =
-                std::max(_geometry.sectorSize, (unsigned)sector->data.size());
+                std::max(_geometry.sectorSize, sector->data.size());
+            _geometry.totalBytes += _geometry.sectorSize;
         }
     }
     _geometry.numSectors = maxSector - _geometry.firstSector + 1;
 }
+
+void Image::populateSectorPhysicalLocationsFromLogicalLocations(
+    const DiskLayout& diskLayout)
+{
+    Image tempImage;
+    for (const auto& sector : *this)
+    {
+        const auto& ltl = diskLayout.layoutByLogicalLocation.at(
+            {sector->logicalCylinder, sector->logicalHead});
+        auto newSector = tempImage.put(sector->logicalCylinder,
+            sector->logicalHead,
+            sector->logicalSector);
+        *newSector = *sector;
+        newSector->physicalLocation = std::make_optional<CylinderHead>(
+            ltl->physicalCylinder, ltl->physicalHead);
+    }
+
+    for (const auto& sector : tempImage)
+        _sectors[{sector->logicalCylinder,
+            sector->logicalHead,
+            sector->logicalSector}] = sector;
+}

lib/data/image.h

@@ -1,32 +1,33 @@
 #ifndef IMAGE_H
 #define IMAGE_H
 
+#include "lib/data/locations.h"
+
 class Sector;
+class DiskLayout;
 
 struct Geometry
 {
-    unsigned numTracks = 0;
+    unsigned numCylinders = 0;
-    unsigned numSides = 0;
+    unsigned numHeads = 0;
     unsigned firstSector = UINT_MAX;
     unsigned numSectors = 0;
     unsigned sectorSize = 0;
     bool irregular = false;
+    unsigned totalBytes = 0;
 };
 
 class Image
 {
-private:
-    typedef std::tuple<unsigned, unsigned, unsigned> key_t;
-
 public:
     Image();
-    Image(std::set<std::shared_ptr<const Sector>>& sectors);
+    Image(const std::vector<std::shared_ptr<const Sector>>& sectors);
 
 public:
     class const_iterator
     {
-        typedef std::map<key_t, std::shared_ptr<const Sector>>::const_iterator
+        typedef std::map<LogicalLocation,
-            wrapped_iterator_t;
+            std::shared_ptr<const Sector>>::const_iterator wrapped_iterator_t;
 
     public:
         const_iterator(const wrapped_iterator_t& it): _it(it) {}
@@ -60,14 +61,53 @@ public:
     void clear();
     void createBlankImage();
     bool empty() const;
-    bool contains(unsigned track, unsigned side, unsigned sectorId) const;
-    std::shared_ptr<const Sector> get(
-        unsigned track, unsigned side, unsigned sectorId) const;
-    std::shared_ptr<Sector> put(
-        unsigned track, unsigned side, unsigned sectorId);
-    void erase(unsigned track, unsigned side, unsigned sectorId);
 
-    std::set<std::pair<unsigned, unsigned>> tracks() const;
+    bool contains(const LogicalLocation& location) const;
+    std::shared_ptr<const Sector> get(const LogicalLocation& location) const;
+    std::shared_ptr<Sector> put(const LogicalLocation& location);
+    void erase(const LogicalLocation& location);
+
+    bool contains(const CylinderHead& ch, unsigned sector) const
+    {
+        return contains({ch.cylinder, ch.head, sector});
+    }
+
+    bool contains(unsigned cylinder, unsigned head, unsigned sector) const
+    {
+        return contains({cylinder, head, sector});
+    }
+
+    std::shared_ptr<const Sector> get(
+        const CylinderHead& ch, unsigned sector) const
+    {
+        return get({ch.cylinder, ch.head, sector});
+    }
+
+    std::shared_ptr<const Sector> get(
+        unsigned cylinder, unsigned head, unsigned sector) const
+    {
+        return get({cylinder, head, sector});
+    }
+
+    std::shared_ptr<Sector> put(const CylinderHead& ch, unsigned sector)
+    {
+        return put({ch.cylinder, ch.head, sector});
+    }
+
+    std::shared_ptr<Sector> put(
+        unsigned cylinder, unsigned head, unsigned sector)
+    {
+        return put({cylinder, head, sector});
+    }
+
+    void erase(unsigned cylinder, unsigned head, unsigned sector)
+    {
+        erase({cylinder, head, sector});
+    }
+
+    void addMissingSectors(const DiskLayout& layout, bool populated = false);
+    void populateSectorPhysicalLocationsFromLogicalLocations(
+        const DiskLayout& diskLayout);
 
     const_iterator begin() const
     {
@@ -89,7 +129,7 @@ public:
 
 private:
     Geometry _geometry = {0, 0, 0};
-    std::map<key_t, std::shared_ptr<const Sector>> _sectors;
+    std::map<LogicalLocation, std::shared_ptr<const Sector>> _sectors;
 };
 
 #endif

lib/data/layout.cc

@@ -4,10 +4,10 @@
 #include "lib/config/proto.h"
 #include "lib/core/logger.h"
 
-static unsigned getTrackStep()
+static unsigned getTrackStep(const ConfigProto& config = globalConfig())
 {
-    auto format_type = globalConfig()->layout().format_type();
+    auto format_type = config.layout().format_type();
-    auto drive_type = globalConfig()->drive().drive_type();
+    auto drive_type = config.drive().drive_type();
 
     switch (format_type)
     {
@@ -22,6 +22,9 @@ static unsigned getTrackStep()
 
                 case DRIVETYPE_APPLE2:
                     return 4;
+
+                default:
+                    break;
             }
 
         case FORMATTYPE_80TRACK:
@@ -30,8 +33,7 @@ static unsigned getTrackStep()
                 case DRIVETYPE_40TRACK:
                     error(
                         "you can't read/write an 80 track image from/to a 40 "
-                        "track "
+                        "track drive");
-                        "drive");
 
                 case DRIVETYPE_80TRACK:
                     return 1;
@@ -39,111 +41,55 @@ static unsigned getTrackStep()
                 case DRIVETYPE_APPLE2:
                     error(
                         "you can't read/write an 80 track image from/to an "
-                        "Apple II "
+                        "Apple II drive");
-                        "drive");
+
+                default:
+                    break;
             }
+
+        default:
+            break;
     }
 
     return 1;
 }
 
-unsigned Layout::remapTrackPhysicalToLogical(unsigned ptrack)
+static std::vector<CylinderHead> getTrackOrdering(
+    LayoutProto::Order ordering, unsigned tracks, unsigned sides)
 {
-    return (ptrack - globalConfig()->drive().head_bias()) / getTrackStep();
+    std::vector<CylinderHead> trackList;
-}
-
-unsigned Layout::remapTrackLogicalToPhysical(unsigned ltrack)
-{
-    return globalConfig()->drive().head_bias() + ltrack * getTrackStep();
-}
-
-unsigned Layout::remapSidePhysicalToLogical(unsigned pside)
-{
-    return pside ^ globalConfig()->layout().swap_sides();
-}
-
-unsigned Layout::remapSideLogicalToPhysical(unsigned lside)
-{
-    return lside ^ globalConfig()->layout().swap_sides();
-}
-
-std::vector<CylinderHead> Layout::computePhysicalLocations()
-{
-    if (globalConfig()->has_tracks())
-        return parseCylinderHeadsString(globalConfig()->tracks());
-
-    std::set<unsigned> tracks = iterate(0, globalConfig()->layout().tracks());
-    std::set<unsigned> heads = iterate(0, globalConfig()->layout().sides());
-
-    std::vector<CylinderHead> locations;
-    for (unsigned logicalTrack : tracks)
-        for (unsigned logicalHead : heads)
-            locations.push_back(
-                CylinderHead{remapTrackLogicalToPhysical(logicalTrack),
-                    remapSideLogicalToPhysical(logicalHead)});
-
-    return locations;
-}
-
-Layout::LayoutBounds Layout::getBounds(
-    const std::vector<CylinderHead>& locations)
-{
-    LayoutBounds r{.minTrack = INT_MAX,
-        .maxTrack = INT_MIN,
-        .minSide = INT_MAX,
-        .maxSide = INT_MIN};
-
-    for (const auto& ti : locations)
-    {
-        r.minTrack = std::min<int>(r.minTrack, ti.cylinder);
-        r.maxTrack = std::max<int>(r.maxTrack, ti.cylinder);
-        r.minSide = std::min<int>(r.minSide, ti.head);
-        r.maxSide = std::max<int>(r.maxSide, ti.head);
-    }
-
-    return r;
-}
-
-std::vector<std::pair<int, int>> Layout::getTrackOrdering(
-    LayoutProto::Order ordering, unsigned guessedTracks, unsigned guessedSides)
-{
-    auto layout = globalConfig()->layout();
-    int tracks = layout.has_tracks() ? layout.tracks() : guessedTracks;
-    int sides = layout.has_sides() ? layout.sides() : guessedSides;
-
-    std::vector<std::pair<int, int>> trackList;
     switch (ordering)
     {
         case LayoutProto::CHS:
         {
-            for (int track = 0; track < tracks; track++)
+            for (unsigned track = 0; track < tracks; track++)
             {
-                for (int side = 0; side < sides; side++)
+                for (unsigned side = 0; side < sides; side++)
-                    trackList.push_back(std::make_pair(track, side));
+                    trackList.push_back({track, side});
             }
             break;
         }
 
         case LayoutProto::HCS:
         {
-            for (int side = 0; side < sides; side++)
+            for (unsigned side = 0; side < sides; side++)
             {
-                for (int track = 0; track < tracks; track++)
+                for (unsigned track = 0; track < tracks; track++)
-                    trackList.push_back(std::make_pair(track, side));
+                    trackList.push_back({track, side});
             }
             break;
         }
 
         case LayoutProto::HCS_RH1:
         {
-            for (int side = 0; side < sides; side++)
+            for (unsigned side = 0; side < sides; side++)
             {
                 if (side == 0)
-                    for (int track = 0; track < tracks; track++)
+                    for (unsigned track = 0; track < tracks; track++)
-                        trackList.push_back(std::make_pair(track, side));
+                        trackList.push_back({track, side});
                 if (side == 1)
-                    for (int track = tracks; track >= 0; track--)
+                    for (unsigned track = tracks; track >= 0; track--)
-                        trackList.push_back(std::make_pair(track - 1, side));
+                        trackList.push_back({track - 1, side});
             }
             break;
         }
@@ -155,7 +101,7 @@ std::vector<std::pair<int, int>> Layout::getTrackOrdering(
     return trackList;
 }
 
-std::vector<unsigned> Layout::expandSectorList(
+static std::vector<unsigned> expandSectorList(
     const SectorListProto& sectorsProto)
 {
     std::vector<unsigned> sectors;
@@ -197,95 +143,200 @@ std::vector<unsigned> Layout::expandSectorList(
     return sectors;
 }
 
-std::shared_ptr<const TrackInfo> Layout::getLayoutOfTrack(
+static const LayoutProto::LayoutdataProto getLayoutData(
-    unsigned logicalTrack, unsigned logicalSide)
+    unsigned logicalCylinder, unsigned logicalHead, const ConfigProto& config)
 {
-    auto trackInfo = std::make_shared<TrackInfo>();
+    LayoutProto::LayoutdataProto layoutData;
-
+    for (const auto& f : config.layout().layoutdata())
-    LayoutProto::LayoutdataProto layoutdata;
-    for (const auto& f : globalConfig()->layout().layoutdata())
     {
         if (f.has_track() && f.has_up_to_track() &&
-            ((logicalTrack < f.track()) || (logicalTrack > f.up_to_track())))
+            ((logicalCylinder < f.track()) ||
+                (logicalCylinder > f.up_to_track())))
             continue;
         if (f.has_track() && !f.has_up_to_track() &&
-            (logicalTrack != f.track()))
+            (logicalCylinder != f.track()))
             continue;
-        if (f.has_side() && (f.side() != logicalSide))
+        if (f.has_side() && (f.side() != logicalHead))
             continue;
 
-        layoutdata.MergeFrom(f);
+        layoutData.MergeFrom(f);
     }
-
+    return layoutData;
-    trackInfo->numTracks = globalConfig()->layout().tracks();
-    trackInfo->numSides = globalConfig()->layout().sides();
-    trackInfo->sectorSize = layoutdata.sector_size();
-    trackInfo->logicalTrack = logicalTrack;
-    trackInfo->logicalSide = logicalSide;
-    trackInfo->physicalTrack = remapTrackLogicalToPhysical(logicalTrack);
-    trackInfo->physicalSide =
-        logicalSide ^ globalConfig()->layout().swap_sides();
-    trackInfo->groupSize = getTrackStep();
-    trackInfo->diskSectorOrder = expandSectorList(layoutdata.physical());
-    trackInfo->naturalSectorOrder = trackInfo->diskSectorOrder;
-    std::sort(trackInfo->naturalSectorOrder.begin(),
-        trackInfo->naturalSectorOrder.end());
-    trackInfo->numSectors = trackInfo->naturalSectorOrder.size();
-
-    if (layoutdata.has_filesystem())
-    {
-        trackInfo->filesystemSectorOrder =
-            expandSectorList(layoutdata.filesystem());
-        if (trackInfo->filesystemSectorOrder.size() != trackInfo->numSectors)
-            error(
-                "filesystem sector order list doesn't contain the right "
-                "number of sectors");
-    }
-    else
-        trackInfo->filesystemSectorOrder = trackInfo->naturalSectorOrder;
-
-    for (int i = 0; i < trackInfo->numSectors; i++)
-    {
-        unsigned fid = trackInfo->naturalSectorOrder[i];
-        unsigned lid = trackInfo->filesystemSectorOrder[i];
-        trackInfo->filesystemToNaturalSectorMap[fid] = lid;
-        trackInfo->naturalToFilesystemSectorMap[lid] = fid;
-    }
-
-    return trackInfo;
 }
 
-std::shared_ptr<const TrackInfo> Layout::getLayoutOfTrackPhysical(
+DiskLayout::DiskLayout(const ConfigProto& config)
-    unsigned physicalTrack, unsigned physicalSide)
 {
-    return getLayoutOfTrack(remapTrackPhysicalToLogical(physicalTrack),
+    minPhysicalCylinder = minPhysicalHead = UINT_MAX;
-        remapSidePhysicalToLogical(physicalSide));
+    maxPhysicalCylinder = maxPhysicalHead = 0;
-}
 
-std::shared_ptr<const TrackInfo> Layout::getLayoutOfTrackPhysical(
+    numLogicalCylinders = config.layout().tracks();
-    const CylinderHead& physicalLocation)
+    numLogicalHeads = config.layout().sides();
-{
-    return getLayoutOfTrackPhysical(
-        physicalLocation.cylinder, physicalLocation.head);
-}
 
-std::vector<std::shared_ptr<const TrackInfo>> Layout::getLayoutOfTracksPhysical(
+    groupSize = getTrackStep(config);
-    const std::vector<CylinderHead>& physicalLocations)
+    headBias = config.drive().head_bias();
-{
+    swapSides = config.layout().swap_sides();
-    std::vector<std::shared_ptr<const TrackInfo>> results;
-    for (const auto& physicalLocation : physicalLocations)
-        results.push_back(getLayoutOfTrackPhysical(physicalLocation));
-    return results;
-}
 
-int Layout::getHeadWidth()
+    switch (config.drive().drive_type())
-{
-    switch (globalConfig()->drive().drive_type())
     {
         case DRIVETYPE_APPLE2:
-            return 4;
+            headWidth = 4;
+            break;
 
         default:
-            return 1;
+            headWidth = 1;
+            break;
     }
+
+    for (unsigned logicalCylinder = 0; logicalCylinder < numLogicalCylinders;
+        logicalCylinder++)
+        for (unsigned logicalHead = 0; logicalHead < numLogicalHeads;
+            logicalHead++)
+        {
+            auto ltl = std::make_shared<LogicalTrackLayout>();
+            CylinderHead ch(logicalCylinder, logicalHead);
+            layoutByLogicalLocation[ch] = ltl;
+            logicalLocations.push_back(ch);
+
+            ltl->logicalCylinder = logicalCylinder;
+            ltl->logicalHead = logicalHead;
+            ltl->groupSize = groupSize;
+            ltl->physicalCylinder =
+                remapCylinderLogicalToPhysical(logicalCylinder);
+            ltl->physicalHead = remapHeadLogicalToPhysical(logicalHead);
+
+            minPhysicalCylinder =
+                std::min(minPhysicalCylinder, ltl->physicalCylinder);
+            maxPhysicalCylinder = std::max(maxPhysicalCylinder,
+                ltl->physicalCylinder + ltl->groupSize - 1);
+            minPhysicalHead = std::min(minPhysicalHead, ltl->physicalHead);
+            maxPhysicalHead = std::max(maxPhysicalHead, ltl->physicalHead);
+
+            auto layoutdata =
+                getLayoutData(logicalCylinder, logicalHead, config);
+            ltl->sectorSize = layoutdata.sector_size();
+            ltl->diskSectorOrder = expandSectorList(layoutdata.physical());
+            ltl->naturalSectorOrder = ltl->diskSectorOrder;
+            std::sort(
+                ltl->naturalSectorOrder.begin(), ltl->naturalSectorOrder.end());
+            ltl->numSectors = ltl->naturalSectorOrder.size();
+
+            if (layoutdata.has_filesystem())
+            {
+                ltl->filesystemSectorOrder =
+                    expandSectorList(layoutdata.filesystem());
+                if (ltl->filesystemSectorOrder.size() != ltl->numSectors)
+                    error(
+                        "filesystem sector order list doesn't contain the "
+                        "right number of sectors");
+            }
+            else
+                ltl->filesystemSectorOrder = ltl->naturalSectorOrder;
+
+            for (int i = 0; i < ltl->numSectors; i++)
+            {
+                unsigned fid = ltl->naturalSectorOrder[i];
+                unsigned lid = ltl->filesystemSectorOrder[i];
+                ltl->sectorIdToNaturalOrdering[i] = fid;
+                ltl->sectorIdToFilesystemOrdering[i] = fid;
+            }
+        };
+
+    for (unsigned physicalCylinder = minPhysicalCylinder;
+        physicalCylinder <= maxPhysicalCylinder;
+        physicalCylinder++)
+        for (unsigned physicalHead = minPhysicalHead;
+            physicalHead <= maxPhysicalHead;
+            physicalHead++)
+        {
+            auto ptl = std::make_shared<PhysicalTrackLayout>();
+            CylinderHead ch(physicalCylinder, physicalHead);
+            layoutByPhysicalLocation[ch] = ptl;
+            physicalLocations.push_back(ch);
+
+            ptl->physicalCylinder = physicalCylinder;
+            ptl->physicalHead = physicalHead;
+            ptl->groupOffset = (physicalCylinder - headBias) % groupSize;
+
+            unsigned logicalCylinder =
+                remapCylinderPhysicalToLogical(physicalCylinder);
+            unsigned logicalHead = remapHeadPhysicalToLogical(physicalHead);
+            ptl->logicalTrackLayout = findOrDefault(
+                layoutByLogicalLocation, {logicalCylinder, logicalHead});
+        }
+
+    unsigned sectorOffset = 0;
+    unsigned blockId = 0;
+    for (auto& ch : getTrackOrdering(config.layout().filesystem_track_order(),
+             numLogicalCylinders,
+             numLogicalHeads))
+    {
+        const auto& ltl = layoutByLogicalLocation[ch];
+        logicalLocationsInFilesystemOrder.push_back(ch);
+
+        for (unsigned lid : ltl->filesystemSectorOrder)
+        {
+            LogicalLocation logicalLocation = {ch.cylinder, ch.head, lid};
+            logicalSectorLocationBySectorOffset[sectorOffset] = logicalLocation;
+            sectorOffsetByLogicalSectorLocation[logicalLocation] = sectorOffset;
+            logicalSectorLocationsInFilesystemOrder.push_back(logicalLocation);
+            sectorOffset += ltl->sectorSize;
+
+            blockIdByLogicalSectorLocation[logicalLocation] = blockId;
+            blockId++;
+        }
+    }
+
+    totalBytes = sectorOffset;
+}
+
+static ConfigProto createTestConfig(unsigned numCylinders,
+    unsigned numHeads,
+    unsigned numSectors,
+    unsigned sectorSize)
+{
+    ConfigProto config;
+    auto* layout = config.mutable_layout();
+    layout->set_tracks(numCylinders);
+    layout->set_sides(numHeads);
+    auto* layoutData = layout->add_layoutdata();
+    layoutData->set_sector_size(sectorSize);
+    layoutData->mutable_physical()->set_count(numSectors);
+
+    return config;
+}
+
+DiskLayout::DiskLayout(unsigned numCylinders,
+    unsigned numHeads,
+    unsigned numSectors,
+    unsigned sectorSize):
+    DiskLayout(createTestConfig(numCylinders, numHeads, numSectors, sectorSize))
+{
+}
+
+static DiskLayout::LayoutBounds getBounds(std::ranges::view auto keys)
+{
+    DiskLayout::LayoutBounds r{.minCylinder = INT_MAX,
+        .maxCylinder = INT_MIN,
+        .minHead = INT_MAX,
+        .maxHead = INT_MIN};
+
+    for (const auto& ch : keys)
+    {
+        r.minCylinder = std::min<int>(r.minCylinder, ch.cylinder);
+        r.maxCylinder = std::max<int>(r.maxCylinder, ch.cylinder);
+        r.minHead = std::min<int>(r.minHead, ch.head);
+        r.maxHead = std::max<int>(r.maxHead, ch.head);
+    }
+
+    return r;
+}
+
+DiskLayout::LayoutBounds DiskLayout::getPhysicalBounds() const
+{
+    return getBounds(std::views::keys(layoutByPhysicalLocation));
+}
+
+DiskLayout::LayoutBounds DiskLayout::getLogicalBounds() const
+{
+    return getBounds(std::views::keys(layoutByLogicalLocation));
 }

lib/data/layout.h

@@ -1,73 +1,163 @@
 #ifndef LAYOUT_H
 #define LAYOUT_H
 
-#include "lib/data/flux.h"
+#include "lib/data/disk.h"
 #include "lib/config/layout.pb.h"
+#include "lib/config/config.h"
 #include "lib/data/locations.h"
 
-class SectorListProto;
+class ConfigProto;
-class TrackInfo;
 
-class Layout
+struct LogicalTrackLayout
+{
+    /* Physical cylinder of the first element of the group. */
+    unsigned physicalCylinder;
+
+    /* Physical head of the first element of the group. */
+    unsigned physicalHead;
+
+    /* Size of this group. */
+    unsigned groupSize;
+
+    /* Logical cylinder of this track. */
+    unsigned logicalCylinder = 0;
+
+    /* Logical side of this track. */
+    unsigned logicalHead = 0;
+
+    /* The number of sectors in this track. */
+    unsigned numSectors = 0;
+
+    /* Number of bytes in a sector. */
+    unsigned sectorSize = 0;
+
+    /* Sector IDs in sector ID order. This is the order in which the appear in
+     * disk images. */
+    std::vector<unsigned> naturalSectorOrder;
+
+    /* Sector IDs in disk order. This is the order they are written to the disk.
+     */
+    std::vector<unsigned> diskSectorOrder;
+
+    /* Sector IDs in filesystem order. This is the order in which the filesystem
+     * uses them. */
+    std::vector<unsigned> filesystemSectorOrder;
+
+    /* Mapping of sector ID to filesystem ordering. */
+    std::map<unsigned, unsigned> sectorIdToFilesystemOrdering;
+
+    /* Mapping of sector ID to natural ordering. */
+    std::map<unsigned, unsigned> sectorIdToNaturalOrdering;
+};
+
+struct PhysicalTrackLayout
+{
+    /* Physical location of this track. */
+    unsigned physicalCylinder;
+
+    /* Physical side of this track. */
+    unsigned physicalHead;
+
+    /* Which member of the group this is. */
+    unsigned groupOffset;
+
+    /* The logical track that this track is part of. */
+    std::shared_ptr<const LogicalTrackLayout> logicalTrackLayout;
+};
+
+class DiskLayout
 {
 public:
-    /* Translates logical track numbering (the numbers actually written in the
+    DiskLayout(const ConfigProto& config = globalConfig());
-     * sector headers) to the track numbering on the actual drive, taking into
-     * account tpi settings.
-     */
-    static unsigned remapTrackPhysicalToLogical(unsigned physicalTrack);
-    static unsigned remapTrackLogicalToPhysical(unsigned logicalTrack);
 
-    /* Translates logical side numbering (the numbers actually written in the
+    /* Makes a simplified layout for testing. */
-     * sector headers) to the sides used on the actual drive.
-     */
-    static unsigned remapSidePhysicalToLogical(unsigned physicalSide);
-    static unsigned remapSideLogicalToPhysical(unsigned logicalSide);
 
-    /* Uses the layout and current track and heads settings to determine
+    DiskLayout(unsigned numCylinders,
-     * which physical tracks are going to be read from or written to.
+        unsigned numHeads,
-     */
+        unsigned numSectors,
-    static std::vector<CylinderHead> computePhysicalLocations();
+        unsigned sectorSize);
+
+public:
+    /* Logical size. */
+
+    unsigned numLogicalCylinders;
+    unsigned numLogicalHeads;
+
+    /* Physical size and properties. */
+
+    unsigned minPhysicalCylinder, maxPhysicalCylinder;
+    unsigned minPhysicalHead, maxPhysicalHead;
+    unsigned groupSize;  /* Number of physical cylinders per logical cylinder */
+    unsigned headBias;   /* Physical cylinder offset */
+    unsigned headWidth;  /* Width of the physical head */
+    bool swapSides;      /* Whether sides need to be swapped */
+    unsigned totalBytes; /* Total number of bytes on the disk. */
+
+    /* Physical and logical layouts by location. */
+
+    std::map<CylinderHead, std::shared_ptr<const PhysicalTrackLayout>>
+        layoutByPhysicalLocation;
+    std::map<CylinderHead, std::shared_ptr<const LogicalTrackLayout>>
+        layoutByLogicalLocation;
+
+    /* Ordered lists of physical and logical locations. */
+
+    std::vector<CylinderHead> logicalLocations;
+    std::vector<CylinderHead> logicalLocationsInFilesystemOrder;
+    std::vector<CylinderHead> physicalLocations;
+
+    /* Ordered lists of sector locations, plus the reverse mapping. */
+
+    std::vector<LogicalLocation> logicalSectorLocationsInFilesystemOrder;
+    std::map<LogicalLocation, unsigned> blockIdByLogicalSectorLocation;
+    std::vector<CylinderHeadSector> physicalSectorLocationsInFilesystemOrder;
+
+    /* Mapping from logical location to sector offset and back again. */
+
+    std::map<unsigned, LogicalLocation> logicalSectorLocationBySectorOffset;
+    std::map<LogicalLocation, unsigned> sectorOffsetByLogicalSectorLocation;
+
+public:
+    unsigned remapCylinderPhysicalToLogical(unsigned physicalCylinder) const
+    {
+        return (physicalCylinder - headBias) / groupSize;
+    }
+
+    unsigned remapCylinderLogicalToPhysical(unsigned logicalCylinder) const
+    {
+        return headBias + logicalCylinder * groupSize;
+    }
+
+    unsigned remapHeadPhysicalToLogical(unsigned physicalHead) const
+    {
+        return physicalHead ^ swapSides;
+    }
+
+    unsigned remapHeadLogicalToPhysical(unsigned logicalHead) const
+    {
+        return logicalHead ^ swapSides;
+    }
 
     /* Given a list of CylinderHead locations, determines the minimum and
      * maximum track and side settings. */
     struct LayoutBounds
     {
-        int minTrack, maxTrack, minSide, maxSide;
+        std::strong_ordering operator<=>(
+            const LayoutBounds& other) const = default;
+
+        int minCylinder, maxCylinder, minHead, maxHead;
     };
-    static LayoutBounds getBounds(const std::vector<CylinderHead>& locations);
 
-    /* Returns a series of <track, side> pairs representing the filesystem
+    LayoutBounds getPhysicalBounds() const;
-     * ordering of the disk, in logical numbers. */
+    LayoutBounds getLogicalBounds() const;
-    static std::vector<std::pair<int, int>> getTrackOrdering(
-        LayoutProto::Order ordering,
-        unsigned guessedTracks = 0,
-        unsigned guessedSides = 0);
-
-    /* Returns the layout of a given track. */
-    static std::shared_ptr<const TrackInfo> getLayoutOfTrack(
-        unsigned logicalTrack, unsigned logicalHead);
-
-    /* Returns the layout of a given track via physical location. */
-    static std::shared_ptr<const TrackInfo> getLayoutOfTrackPhysical(
-        unsigned physicalTrack, unsigned physicalSide);
-
-    /* Returns the layout of a given track via physical location. */
-    static std::shared_ptr<const TrackInfo> getLayoutOfTrackPhysical(
-        const CylinderHead& physicalLocation);
-
-    /* Returns the layouts of a multiple tracks via physical location. */
-    static std::vector<std::shared_ptr<const TrackInfo>>
-    getLayoutOfTracksPhysical(const std::vector<CylinderHead>& locations);
-
-    /* Expand a SectorList into the actual sector IDs. */
-    static std::vector<unsigned> expandSectorList(
-        const SectorListProto& sectorsProto);
-
-    /* Return the head width of the current drive. */
-    static int getHeadWidth();
 };
 
+static std::shared_ptr<DiskLayout> createDiskLayout(
+    const ConfigProto& config = globalConfig())
+{
+    return std::make_shared<DiskLayout>(config);
+}
+
 class TrackInfo
 {
 public:
@@ -79,23 +169,23 @@ private:
     TrackInfo& operator=(const TrackInfo&);
 
 public:
-    unsigned numTracks = 0;
+    unsigned numCylinders = 0;
-    unsigned numSides = 0;
+    unsigned numHeads = 0;
 
     /* The number of sectors in this track. */
     unsigned numSectors = 0;
 
     /* Physical location of this track. */
-    unsigned physicalTrack = 0;
+    unsigned physicalCylinder = 0;
 
     /* Physical side of this track. */
-    unsigned physicalSide = 0;
+    unsigned physicalHead = 0;
 
     /* Logical location of this track. */
-    unsigned logicalTrack = 0;
+    unsigned logicalCylinder = 0;
 
     /* Logical side of this track. */
-    unsigned logicalSide = 0;
+    unsigned logicalHead = 0;
 
     /* The number of physical tracks which need to be written for one logical
      * track. */

lib/data/locations.cc

@@ -10,6 +10,7 @@
 #include <lexy_ext/report_error.hpp>
 #include "fmt/ranges.h"
 #include <ranges>
+#include <algorithm>
 
 namespace
 {

lib/data/locations.h

@@ -8,6 +8,41 @@ struct CylinderHead
     unsigned cylinder, head;
 };
 
+struct CylinderHeadSector
+{
+    bool operator==(const CylinderHeadSector&) const = default;
+    std::strong_ordering operator<=>(const CylinderHeadSector&) const = default;
+
+    unsigned cylinder, head, sector;
+};
+
+struct LogicalLocation
+{
+    bool operator==(const LogicalLocation&) const = default;
+    std::strong_ordering operator<=>(const LogicalLocation&) const = default;
+
+    unsigned logicalCylinder;
+    unsigned logicalHead;
+    unsigned logicalSector;
+
+    operator std::string() const
+    {
+        return fmt::format(
+            "c{}h{}s{}", logicalCylinder, logicalHead, logicalSector);
+    }
+
+    CylinderHead trackLocation() const
+    {
+        return {logicalCylinder, logicalHead};
+    }
+};
+
+inline std::ostream& operator<<(std::ostream& stream, LogicalLocation location)
+{
+    stream << (std::string)location;
+    return stream;
+}
+
 extern std::vector<CylinderHead> parseCylinderHeadsString(const std::string& s);
 extern std::string convertCylinderHeadsToString(
     const std::vector<CylinderHead>& chs);

lib/data/sector.cc

@@ -1,21 +1,9 @@
 #include "lib/core/globals.h"
-#include "lib/data/flux.h"
+#include "lib/data/disk.h"
 #include "lib/data/sector.h"
 #include "lib/data/layout.h"
 
-Sector::Sector(const LogicalLocation& location):
+Sector::Sector(const LogicalLocation& location): LogicalLocation(location) {}
-    LogicalLocation(location),
-    physicalTrack(Layout::remapTrackLogicalToPhysical(location.logicalTrack)),
-    physicalSide(Layout::remapSideLogicalToPhysical(location.logicalSide))
-{
-}
-
-Sector::Sector(std::shared_ptr<const TrackInfo>& layout, unsigned sectorId):
-    LogicalLocation({layout->logicalTrack, layout->logicalSide, sectorId}),
-    physicalTrack(layout->physicalTrack),
-    physicalSide(layout->physicalSide)
-{
-}
 
 std::string Sector::statusToString(Status status)
 {
@@ -32,7 +20,7 @@ std::string Sector::statusToString(Status status)
         case Status::CONFLICT:
             return "conflicting data";
         default:
-            return fmt::format("unknown error {}", status);
+            return fmt::format("unknown error {}", (int)status);
     }
 }
 

lib/data/sector.h

@@ -3,36 +3,10 @@
 
 #include "lib/core/bytes.h"
 #include "lib/data/fluxmap.h"
+#include "lib/data/locations.h"
 
 class Record;
-class TrackInfo;
+class LogicalTrackLayout;
-
-struct LogicalLocation
-{
-    unsigned logicalTrack;
-    unsigned logicalSide;
-    unsigned logicalSector;
-
-    std::tuple<int, int, int> key() const
-    {
-        return std::make_tuple(logicalTrack, logicalSide, logicalSector);
-    }
-
-    bool operator==(const LogicalLocation& rhs) const
-    {
-        return key() == rhs.key();
-    }
-
-    bool operator!=(const LogicalLocation& rhs) const
-    {
-        return key() != rhs.key();
-    }
-
-    bool operator<(const LogicalLocation& rhs) const
-    {
-        return key() < rhs.key();
-    }
-};
 
 struct Sector : public LogicalLocation
 {
@@ -57,36 +31,24 @@ struct Sector : public LogicalLocation
     nanoseconds_t headerEndTime = 0;
     nanoseconds_t dataStartTime = 0;
     nanoseconds_t dataEndTime = 0;
-    unsigned physicalTrack = 0;
+    std::optional<CylinderHead> physicalLocation = {};
-    unsigned physicalSide = 0;
     Bytes data;
     std::vector<std::shared_ptr<Record>> records;
 
-    Sector() {}
+    Sector(const Sector& other) = default;
-
+    Sector& operator=(const Sector& other) = default;
-    Sector(std::shared_ptr<const TrackInfo>& layout, unsigned sectorId = 0);
 
     Sector(const LogicalLocation& location);
 
     std::tuple<int, int, int, Status> key() const
     {
         return std::make_tuple(
-            logicalTrack, logicalSide, logicalSector, status);
+            logicalCylinder, logicalHead, logicalSector, status);
     }
 
-    bool operator==(const Sector& rhs) const
+    std::strong_ordering operator<=>(const Sector& rhs) const
     {
-        return key() == rhs.key();
+        return key() <=> rhs.key();
-    }
-
-    bool operator!=(const Sector& rhs) const
-    {
-        return key() != rhs.key();
-    }
-
-    bool operator<(const Sector& rhs) const
-    {
-        return key() < rhs.key();
     }
 };
 
@@ -95,7 +57,7 @@ struct fmt::formatter<Sector::Status> : formatter<string_view>
 {
     auto format(Sector::Status status, format_context& ctx) const
     {
-        return format_to(ctx.out(), "{}", Sector::statusToString(status));
+        return fmt::format_to(ctx.out(), "{}", Sector::statusToString(status));
     }
 };
 

lib/decoders/decoders.cc

@@ -4,7 +4,7 @@
 #include "lib/config/config.h"
 #include "lib/decoders/decoders.h"
 #include "lib/data/fluxmapreader.h"
-#include "lib/data/flux.h"
+#include "lib/data/disk.h"
 #include "protocol.h"
 #include "lib/decoders/rawbits.h"
 #include "lib/data/sector.h"
@@ -13,20 +13,25 @@
 #include "lib/data/layout.h"
 #include <numeric>
 
-std::shared_ptr<TrackDataFlux> Decoder::decodeToSectors(
+std::shared_ptr<Track> Decoder::decodeToSectors(
     std::shared_ptr<const Fluxmap> fluxmap,
-    std::shared_ptr<const TrackInfo>& trackInfo)
+    const std::shared_ptr<const PhysicalTrackLayout>& ptl)
 {
-    _trackdata = std::make_shared<TrackDataFlux>();
+    _ltl = ptl->logicalTrackLayout;
+
+    _trackdata = std::make_shared<Track>();
     _trackdata->fluxmap = fluxmap;
-    _trackdata->trackInfo = trackInfo;
+    _trackdata->ptl = ptl;
+    _trackdata->ltl = ptl->logicalTrackLayout;
 
     FluxmapReader fmr(*fluxmap);
     _fmr = &fmr;
 
     auto newSector = [&]
     {
-        _sector = std::make_shared<Sector>(trackInfo, 0);
+        _sector = std::make_shared<Sector>(LogicalLocation{0, 0, 0});
+        _sector->physicalLocation = std::make_optional<CylinderHead>(
+            ptl->physicalCylinder, ptl->physicalHead);
         _sector->status = Sector::MISSING;
     };
 
@@ -67,6 +72,7 @@ std::shared_ptr<TrackDataFlux> Decoder::decodeToSectors(
 
             before = fmr.tell();
             decodeDataRecord();
+            _sector->data = _sector->data.slice(0, _ltl->sectorSize);
             after = fmr.tell();
 
             if (_sector->status != Sector::DATA_MISSING)
@@ -84,11 +90,7 @@ std::shared_ptr<TrackDataFlux> Decoder::decodeToSectors(
         }
 
         if (_sector->status != Sector::MISSING)
-        {
+            _trackdata->allSectors.push_back(_sector);
-            auto trackLayout = Layout::getLayoutOfTrack(
-                _sector->logicalTrack, _sector->logicalSide);
-            _trackdata->sectors.push_back(_sector);
-        }
     }
 
     return _trackdata;

lib/decoders/decoders.h

@@ -6,15 +6,16 @@
 #include "lib/data/fluxmapreader.h"
 #include "lib/decoders/fluxdecoder.h"
 
-class Sector;
-class Fluxmap;
-class FluxMatcher;
-class FluxmapReader;
-class RawBits;
-class DecoderProto;
 class Config;
+class DecoderProto;
+class FluxMatcher;
+class Fluxmap;
+class FluxmapReader;
+class PhysicalTrackLayout;
+class RawBits;
+class Sector;
 
-#include "lib/data/flux.h"
+#include "lib/data/disk.h"
 
 extern void setDecoderManualClockRate(double clockrate_us);
 
@@ -52,9 +53,9 @@ public:
     };
 
 public:
-    std::shared_ptr<TrackDataFlux> decodeToSectors(
+    std::shared_ptr<Track> decodeToSectors(
         std::shared_ptr<const Fluxmap> fluxmap,
-        std::shared_ptr<const TrackInfo>& trackInfo);
+        const std::shared_ptr<const PhysicalTrackLayout>& ptl);
 
     void pushRecord(
         const Fluxmap::Position& start, const Fluxmap::Position& end);
@@ -104,7 +105,8 @@ protected:
     virtual void decodeDataRecord() {};
 
     const DecoderProto& _config;
-    std::shared_ptr<TrackDataFlux> _trackdata;
+    std::shared_ptr<const LogicalTrackLayout> _ltl;
+    std::shared_ptr<Track> _trackdata;
     std::shared_ptr<Sector> _sector;
     std::unique_ptr<FluxDecoder> _decoder;
     std::vector<bool> _recordBits;

lib/encoders/encoders.cc

@@ -5,6 +5,7 @@
 #include "lib/encoders/encoders.pb.h"
 #include "lib/config/proto.h"
 #include "lib/data/layout.h"
+#include "lib/data/locations.h"
 #include "lib/data/image.h"
 #include "protocol.h"
 
@@ -23,25 +24,24 @@ nanoseconds_t Encoder::calculatePhysicalClockPeriod(
 }
 
 std::shared_ptr<const Sector> Encoder::getSector(
-    std::shared_ptr<const TrackInfo>& trackInfo,
+    const CylinderHead& ch, const Image& image, unsigned sectorId)
-    const Image& image,
-    unsigned sectorId)
 {
-    return image.get(trackInfo->logicalTrack, trackInfo->logicalSide, sectorId);
+    return image.get(ch.cylinder, ch.head, sectorId);
 }
 
 std::vector<std::shared_ptr<const Sector>> Encoder::collectSectors(
-    std::shared_ptr<const TrackInfo>& trackLayout, const Image& image)
+    const LogicalTrackLayout& ltl, const Image& image)
 {
     std::vector<std::shared_ptr<const Sector>> sectors;
 
-    for (unsigned sectorId : trackLayout->diskSectorOrder)
+    for (unsigned sectorId : ltl.diskSectorOrder)
     {
-        const auto& sector = getSector(trackLayout, image, sectorId);
+        const auto& sector =
+            getSector({ltl.logicalCylinder, ltl.logicalHead}, image, sectorId);
         if (!sector)
             error("sector {}.{}.{} is missing from the image",
-                trackLayout->logicalTrack,
+                ltl.logicalCylinder,
-                trackLayout->logicalSide,
+                ltl.logicalHead,
                 sectorId);
         sectors.push_back(sector);
     }

lib/encoders/encoders.h

@@ -1,13 +1,14 @@
 #ifndef ENCODERS_H
 #define ENCODERS_H
 
+class Config;
+class CylinderHead;
 class EncoderProto;
 class Fluxmap;
 class Image;
 class Layout;
+class LogicalTrackLayout;
 class Sector;
-class TrackInfo;
-class Config;
 
 class Encoder
 {
@@ -19,15 +20,12 @@ public:
 
 public:
     virtual std::shared_ptr<const Sector> getSector(
-        std::shared_ptr<const TrackInfo>&,
+        const CylinderHead& ch, const Image& image, unsigned sectorId);
-        const Image& image,
-        unsigned sectorId);
 
     virtual std::vector<std::shared_ptr<const Sector>> collectSectors(
-        std::shared_ptr<const TrackInfo>&, const Image& image);
+        const LogicalTrackLayout& ltl, const Image& image);
 
-    virtual std::unique_ptr<Fluxmap> encode(
+    virtual std::unique_ptr<Fluxmap> encode(const LogicalTrackLayout& ltl,
-        std::shared_ptr<const TrackInfo>& trackInfo,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) = 0;
 

lib/external/kryoflux.cc

@@ -28,14 +28,6 @@ std::unique_ptr<Fluxmap> readStream(
 {
     std::string suffix = fmt::format("{:02}.{}.raw", track, side);
 
-    FILE* fp = fopen(dir.c_str(), "r");
-    if (fp)
-    {
-        fclose(fp);
-        int i = dir.find_last_of("/\\");
-        dir = dir.substr(0, i);
-    }
-
     DIR* dirp = opendir(dir.c_str());
     if (!dirp)
         error("cannot access path '{}'", dir);

lib/fluxsink/a2rfluxsink.cc

@@ -17,228 +17,246 @@
 #include <sys/types.h>
 #include "fmt/chrono.h"
 
-namespace
+static uint32_t ticks_to_a2r(uint32_t ticks)
 {
-    uint32_t ticks_to_a2r(uint32_t ticks)
+    return ticks * NS_PER_TICK / A2R_NS_PER_TICK;
+}
+
+class A2RSink : public FluxSink
+{
+public:
+    A2RSink(const std::string& filename):
+        _filename(filename),
+        _bytes{},
+        _writer(_bytes.writer())
     {
-        return ticks * NS_PER_TICK / A2R_NS_PER_TICK;
+        time_t now{std::time(nullptr)};
+        auto t = gmtime(&now);
+        _metadata["image_date"] = fmt::format("{:%FT%TZ}", *t);
     }
 
-    class A2RFluxSink : public FluxSink
+    ~A2RSink()
     {
-    public:
+        // FIXME: should use a passed-in DiskLayout object.
-        A2RFluxSink(const A2RFluxSinkProto& lconfig):
+        auto diskLayout = createDiskLayout();
-            _config(lconfig),
+        auto [minCylinder, maxCylinder, minHead, maxHead] =
-            _bytes{},
+            diskLayout->getPhysicalBounds();
-            _writer{_bytes.writer()}
+
+        _minCylinder = minCylinder;
+        _maxCylinder = maxCylinder;
+        _minHead = minHead;
+        _maxHead = maxHead;
+
+        log("A2R: writing A2R {} file containing {} tracks...",
+            (_minHead == _maxHead) ? "single sided" : "double sided",
+            _maxCylinder - _minCylinder + 1);
+
+        writeHeader();
+        writeInfo();
+        writeStream();
+        writeMeta();
+
+        std::ofstream of(_filename, std::ios::out | std::ios::binary);
+        if (!of.is_open())
+            error("cannot open output file");
+        _bytes.writeTo(of);
+        of.close();
+    }
+
+private:
+    void writeChunkAndData(uint32_t chunk_id, const Bytes& data)
+    {
+        _writer.write_le32(chunk_id);
+        _writer.write_le32(data.size());
+        _writer += data;
+    }
+
+    void writeHeader()
+    {
+        static const uint8_t a2r2_fileheader[] = {
+            'A', '2', 'R', '2', 0xff, 0x0a, 0x0d, 0x0a};
+        _writer += Bytes(a2r2_fileheader, sizeof(a2r2_fileheader));
+    }
+
+    void writeInfo()
+    {
+        Bytes info;
+        auto writer = info.writer();
+        writer.write_8(A2R_INFO_CHUNK_VERSION);
+        auto version_str_padded = fmt::format("{: <32}", "FluxEngine");
+        assert(version_str_padded.size() == 32);
+        writer.append(version_str_padded);
+
+        writer.write_8(
+            (globalConfig()->drive().drive_type() == DRIVETYPE_APPLE2)
+                ? A2R_DISK_525
+                : A2R_DISK_35);
+
+        writer.write_8(1); // write protected
+        writer.write_8(1); // synchronized
+        writeChunkAndData(A2R_CHUNK_INFO, info);
+    }
+
+    void writeMeta()
+    {
+        Bytes meta;
+        auto writer = meta.writer();
+        for (auto& i : _metadata)
         {
-            time_t now{std::time(nullptr)};
+            writer.append(i.first);
-            auto t = gmtime(&now);
+            writer.write_8('\t');
-            _metadata["image_date"] = fmt::format("{:%FT%TZ}", *t);
+            writer.append(i.second);
+            writer.write_8('\n');
+        }
+        writeChunkAndData(A2R_CHUNK_META, meta);
+    }
+
+    void writeStream()
+    {
+        // A STRM always ends with a 255, even though this could ALSO
+        // indicate the first byte of a multi-byte sequence
+        _strmWriter.write_8(255);
+
+        writeChunkAndData(A2R_CHUNK_STRM, _strmBytes);
+    }
+
+public:
+    void addFlux(int cylinder, int head, const Fluxmap& fluxmap) override
+    {
+        if (!fluxmap.bytes())
+        {
+            return;
         }
 
-        ~A2RFluxSink()
+        // Writing from an image (as opposed to from a floppy) will
+        // contain exactly one revolution and no index events.
+        auto is_image = [](auto& fluxmap)
         {
-            auto physicalLocations = Layout::computePhysicalLocations();
-            auto [minTrack, maxTrack, minSide, maxSide] =
-                Layout::getBounds(physicalLocations);
-            _minTrack = minTrack;
-            _maxTrack = maxTrack;
-            _minSide = minSide;
-            _maxSide = maxSide;
-
-            log("A2R: writing A2R {} file containing {} tracks...",
-                (_minSide == _maxSide) ? "single sided" : "double sided",
-                _maxTrack - _minTrack + 1);
-
-            writeHeader();
-            writeInfo();
-            writeStream();
-            writeMeta();
-
-            std::ofstream of(
-                _config.filename(), std::ios::out | std::ios::binary);
-            if (!of.is_open())
-                error("cannot open output file");
-            _bytes.writeTo(of);
-            of.close();
-        }
-
-    private:
-        void writeChunkAndData(uint32_t chunk_id, const Bytes& data)
-        {
-            _writer.write_le32(chunk_id);
-            _writer.write_le32(data.size());
-            _writer += data;
-        }
-
-        void writeHeader()
-        {
-            static const uint8_t a2r2_fileheader[] = {
-                'A', '2', 'R', '2', 0xff, 0x0a, 0x0d, 0x0a};
-            _writer += Bytes(a2r2_fileheader, sizeof(a2r2_fileheader));
-        }
-
-        void writeInfo()
-        {
-            Bytes info;
-            auto writer = info.writer();
-            writer.write_8(A2R_INFO_CHUNK_VERSION);
-            auto version_str_padded = fmt::format("{: <32}", "FluxEngine");
-            assert(version_str_padded.size() == 32);
-            writer.append(version_str_padded);
-
-            writer.write_8(
-                (globalConfig()->drive().drive_type() == DRIVETYPE_APPLE2)
-                    ? A2R_DISK_525
-                    : A2R_DISK_35);
-
-            writer.write_8(1); // write protected
-            writer.write_8(1); // synchronized
-            writeChunkAndData(A2R_CHUNK_INFO, info);
-        }
-
-        void writeMeta()
-        {
-            Bytes meta;
-            auto writer = meta.writer();
-            for (auto& i : _metadata)
-            {
-                writer.append(i.first);
-                writer.write_8('\t');
-                writer.append(i.second);
-                writer.write_8('\n');
-            }
-            writeChunkAndData(A2R_CHUNK_META, meta);
-        }
-
-        void writeStream()
-        {
-            // A STRM always ends with a 255, even though this could ALSO
-            // indicate the first byte of a multi-byte sequence
-            _strmWriter.write_8(255);
-
-            writeChunkAndData(A2R_CHUNK_STRM, _strmBytes);
-        }
-
-        void writeFlux(int cylinder, int head, const Fluxmap& fluxmap) override
-        {
-            if (!fluxmap.bytes())
-            {
-                return;
-            }
-
-            // Writing from an image (as opposed to from a floppy) will contain
-            // exactly one revolution and no index events.
-            auto is_image = [](auto& fluxmap)
-            {
-                FluxmapReader fmr(fluxmap);
-                fmr.skipToEvent(F_BIT_INDEX);
-                // but maybe there is no index, if we're writing from an image
-                // to an a2r
-                return fmr.eof();
-            };
-
-            // Write the flux data into its own Bytes
-            Bytes trackBytes;
-            auto trackWriter = trackBytes.writer();
-
-            auto write_one_flux = [&](unsigned ticks)
-            {
-                auto value = ticks_to_a2r(ticks);
-                while (value > 254)
-                {
-                    trackWriter.write_8(255);
-                    value -= 255;
-                }
-                trackWriter.write_8(value);
-            };
-
-            int revolution = 0;
-            uint32_t loopPoint = 0;
-            uint32_t totalTicks = 0;
             FluxmapReader fmr(fluxmap);
+            fmr.skipToEvent(F_BIT_INDEX);
+            // but maybe there is no index, if we're writing from an
+            // image to an a2r
+            return fmr.eof();
+        };
 
-            auto write_flux = [&](unsigned maxTicks = ~0u)
+        // Write the flux data into its own Bytes
-            {
+        Bytes trackBytes;
-                unsigned ticksSinceLastPulse = 0;
+        auto trackWriter = trackBytes.writer();
 
-                while (!fmr.eof() && totalTicks < maxTicks)
+        auto write_one_flux = [&](unsigned ticks)
-                {
-                    unsigned ticks;
-                    int event;
-                    fmr.getNextEvent(event, ticks);
-
-                    ticksSinceLastPulse += ticks;
-                    totalTicks += ticks;
-
-                    if (event & F_BIT_PULSE)
-                    {
-                        write_one_flux(ticksSinceLastPulse);
-                        ticksSinceLastPulse = 0;
-                    }
-
-                    if (event & F_BIT_INDEX && revolution == 0)
-                    {
-                        loopPoint = totalTicks;
-                        revolution += 1;
-                    }
-                }
-            };
-
-            if (is_image(fluxmap))
-            {
-                // A timing stream with no index represents exactly one
-                // revolution with no index. However, a2r nominally contains 450
-                // degress of rotation, 250ms at 300rpm.
-                write_flux();
-                loopPoint = totalTicks;
-                fmr.rewind();
-                revolution += 1;
-                write_flux(totalTicks * 5 / 4);
-            }
-            else
-            {
-                // We have an index, so this is a real read from a floppy and
-                // should be "one revolution plus a bit"
-                fmr.skipToEvent(F_BIT_INDEX);
-                write_flux();
-            }
-
-            uint32_t chunk_size = 10 + trackBytes.size();
-
-            if (globalConfig()->drive().drive_type() == DRIVETYPE_APPLE2)
-                _strmWriter.write_8(cylinder);
-            else
-                _strmWriter.write_8((cylinder << 1) | head);
-
-            _strmWriter.write_8(A2R_TIMING);
-            _strmWriter.write_le32(trackBytes.size());
-            _strmWriter.write_le32(ticks_to_a2r(loopPoint));
-            _strmWriter += trackBytes;
-        }
-
-        operator std::string() const override
         {
-            return fmt::format("a2r({})", _config.filename());
+            auto value = ticks_to_a2r(ticks);
+            while (value > 254)
+            {
+                trackWriter.write_8(255);
+                value -= 255;
+            }
+            trackWriter.write_8(value);
+        };
+
+        int revolution = 0;
+        uint32_t loopPoint = 0;
+        uint32_t totalTicks = 0;
+        FluxmapReader fmr(fluxmap);
+
+        auto write_flux = [&](unsigned maxTicks = ~0u)
+        {
+            unsigned ticksSinceLastPulse = 0;
+
+            while (!fmr.eof() && totalTicks < maxTicks)
+            {
+                unsigned ticks;
+                int event;
+                fmr.getNextEvent(event, ticks);
+
+                ticksSinceLastPulse += ticks;
+                totalTicks += ticks;
+
+                if (event & F_BIT_PULSE)
+                {
+                    write_one_flux(ticksSinceLastPulse);
+                    ticksSinceLastPulse = 0;
+                }
+
+                if (event & F_BIT_INDEX && revolution == 0)
+                {
+                    loopPoint = totalTicks;
+                    revolution += 1;
+                }
+            }
+        };
+
+        if (is_image(fluxmap))
+        {
+            // A timing stream with no index represents exactly one
+            // revolution with no index. However, a2r nominally contains
+            // 450 degress of rotation, 250ms at 300rpm.
+            write_flux();
+            loopPoint = totalTicks;
+            fmr.rewind();
+            revolution += 1;
+            write_flux(totalTicks * 5 / 4);
+        }
+        else
+        {
+            // We have an index, so this is a real read from a floppy
+            // and should be "one revolution plus a bit"
+            fmr.skipToEvent(F_BIT_INDEX);
+            write_flux();
         }
 
-    private:
+        uint32_t chunk_size = 10 + trackBytes.size();
-        const A2RFluxSinkProto& _config;
-        Bytes _bytes;
-        ByteWriter _writer;
-        Bytes _strmBytes;
-        ByteWriter _strmWriter{_strmBytes.writer()};
-        std::map<std::string, std::string> _metadata;
-        int _minSide;
-        int _maxSide;
-        int _minTrack;
-        int _maxTrack;
-    };
-} // namespace
 
-std::unique_ptr<FluxSink> FluxSink::createA2RFluxSink(
+        if (globalConfig()->drive().drive_type() == DRIVETYPE_APPLE2)
+            _strmWriter.write_8(cylinder);
+        else
+            _strmWriter.write_8((cylinder << 1) | head);
+
+        _strmWriter.write_8(A2R_TIMING);
+        _strmWriter.write_le32(trackBytes.size());
+        _strmWriter.write_le32(ticks_to_a2r(loopPoint));
+        _strmWriter += trackBytes;
+    }
+
+private:
+    std::string _filename;
+    Bytes _bytes;
+    ByteWriter _writer;
+    Bytes _strmBytes;
+    ByteWriter _strmWriter{_strmBytes.writer()};
+    std::map<std::string, std::string> _metadata;
+    int _minHead;
+    int _maxHead;
+    int _minCylinder;
+    int _maxCylinder;
+};
+
+class A2RFluxSinkFactory : public FluxSinkFactory
+{
+public:
+    A2RFluxSinkFactory(const A2RFluxSinkProto& lconfig): _config(lconfig) {}
+
+    std::unique_ptr<FluxSink> create() override
+    {
+        return std::make_unique<A2RSink>(_config.filename());
+    }
+
+    std::optional<std::filesystem::path> getPath() const override
+    {
+        return std::make_optional(_config.filename());
+    }
+
+    operator std::string() const override
+    {
+        return fmt::format("a2r({})", _config.filename());
+    }
+
+private:
+    const A2RFluxSinkProto& _config;
+};
+
+std::unique_ptr<FluxSinkFactory> FluxSinkFactory::createA2RFluxSinkFactory(
     const A2RFluxSinkProto& config)
 {
-    return std::unique_ptr<FluxSink>(new A2RFluxSink(config));
+    return std::unique_ptr<FluxSinkFactory>(new A2RFluxSinkFactory(config));
 }

lib/fluxsink/aufluxsink.cc

@@ -1,4 +1,5 @@
 #include "lib/core/globals.h"
+#include "lib/core/logger.h"
 #include "lib/config/flags.h"
 #include "lib/data/fluxmap.h"
 #include "lib/core/bytes.h"
@@ -11,27 +12,29 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
-class AuFluxSink : public FluxSink
+class AuSink : public FluxSink
 {
 public:
-    AuFluxSink(const AuFluxSinkProto& config): _config(config) {}
+    AuSink(const std::string& directory, bool indexMarkers):
-
+        _directory(directory),
-    ~AuFluxSink()
+        _indexMarkers(indexMarkers)
     {
-        std::cerr << "Warning: do not play these files, or you will break your "
-                     "speakers and/or ears!\n";
     }
 
-public:
+    ~AuSink()
-    void writeFlux(int track, int head, const Fluxmap& fluxmap) override
+    {
+        log("Warning: do not play these files, or you will break your "
+            "speakers and/or ears!");
+    }
+
+    void addFlux(int track, int head, const Fluxmap& fluxmap) override
     {
         unsigned totalTicks = fluxmap.ticks() + 2;
-        unsigned channels = _config.index_markers() ? 2 : 1;
+        unsigned channels = _indexMarkers ? 2 : 1;
 
-        mkdir(_config.directory().c_str(), 0744);
+        mkdir(_directory.c_str(), 0744);
         std::ofstream of(
-            fmt::format(
+            fmt::format("{}/c{:02d}.h{:01d}.au", _directory, track, head),
-                "{}/c{:02d}.h{:01d}.au", _config.directory(), track, head),
             std::ios::out | std::ios::binary);
         if (!of.is_open())
             error("cannot open output file");
@@ -73,7 +76,7 @@ public:
 
                 if (event & F_BIT_PULSE)
                     data[timestamp * channels + 0] = 0x7f;
-                if (_config.index_markers() && (event & F_BIT_INDEX))
+                if (_indexMarkers && (event & F_BIT_INDEX))
                     data[timestamp * channels + 1] = 0x7f;
             }
 
@@ -81,6 +84,27 @@ public:
         }
     }
 
+private:
+    std::string _directory;
+    bool _indexMarkers;
+};
+
+class AuFluxSinkFactory : public FluxSinkFactory
+{
+public:
+    AuFluxSinkFactory(const AuFluxSinkProto& config): _config(config) {}
+
+    std::unique_ptr<FluxSink> create() override
+    {
+        return std::make_unique<AuSink>(
+            _config.directory(), _config.index_markers());
+    }
+
+    std::optional<std::filesystem::path> getPath() const override
+    {
+        return std::make_optional(_config.directory());
+    }
+
     operator std::string() const override
     {
         return fmt::format("au({})", _config.directory());
@@ -90,8 +114,8 @@ private:
     const AuFluxSinkProto& _config;
 };
 
-std::unique_ptr<FluxSink> FluxSink::createAuFluxSink(
+std::unique_ptr<FluxSinkFactory> FluxSinkFactory::createAuFluxSinkFactory(
     const AuFluxSinkProto& config)
 {
-    return std::unique_ptr<FluxSink>(new AuFluxSink(config));
+    return std::unique_ptr<FluxSinkFactory>(new AuFluxSinkFactory(config));
 }