Merge pull request #824 from davidgiven/ab

Update ab to the new ninja version.

.clang-format

@@ -18,17 +18,19 @@ AlwaysBreakBeforeMultilineStrings: 'true'
 AlwaysBreakTemplateDeclarations: 'Yes'
 BinPackArguments: 'false'
 BinPackParameters: 'false'
-BreakConstructorInitializers: 'AfterColon'
 BreakBeforeBraces: Allman
+BreakConstructorInitializers: 'AfterColon'
 BreakInheritanceList: AfterColon
 BreakStringLiterals: 'true'
-IndentCaseLabels: 'true'
-IndentWidth: '4'
 ColumnLimit: '80'
 ConstructorInitializerAllOnOneLineOrOnePerLine: 'true'
+FixNamespaceComments: 'false'
 IncludeBlocks: Preserve
+IndentCaseLabels: 'true'
+IndentWidth: '4'
 IndentWrappedFunctionNames: 'false'
 KeepEmptyLinesAtTheStartOfBlocks: 'true'
+NamespaceIndentation: All
 PointerAlignment: Left
 ReflowComments: 'true'
 SortIncludes: 'false'

.github/workflows/ccpp.yml

@@ -2,64 +2,125 @@ name: C/C++ CI
 
 on: [push]
 
+concurrency: 
+  group: environment-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   build-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine'
+        path: 'fluxengine'
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine-testdata'
+        path: 'fluxengine-testdata'
     - name: apt
-      run: sudo apt update && sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.0-gtk3-dev libfmt-dev
+      run: |
+        sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.0-gtk3-dev libfmt-dev libprotobuf-dev
     - name: make
-      run: CXXFLAGS="-Wp,-D_GLIBCXX_ASSERTIONS" make
+      run: CXXFLAGS="-Wp,-D_GLIBCXX_ASSERTIONS" make -j`nproc` -C fluxengine
 
-  build-macos:
-    runs-on: macos-latest
+  #build-linux-debian-11:
+  #  runs-on: ubuntu-22.04
+  #  container: debian:11
+  #  steps:
+  #  - uses: actions/checkout@v4
+  #    with:
+  #      repository: 'davidgiven/fluxengine'
+  #      path: 'fluxengine'
+  #  - uses: actions/checkout@v4
+  #    with:
+  #      repository: 'davidgiven/fluxengine-testdata'
+  #      path: 'fluxengine-testdata'
+  #  - name: apt update
+  #    run: apt update
+  #  - name: apt
+  #    run: >
+  #      apt install -y python3 make xz-utils python3 python3-hamcrest
+  #      protobuf-compiler libprotobuf-dev libsqlite3-dev
+  #      libfmt-dev libprotobuf-dev wx-common pkg-config
+  #      libudev-dev g++ libwxgtk3.0-gtk3-dev
+  #  - name: make
+  #    run: make -C fluxengine
+      
+  build-macos-current:
+    strategy:
+      matrix:
+        runs-on: [macos-13, macos-latest]
+    runs-on: ${{ matrix.runs-on }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine'
+        path: 'fluxengine'
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine-testdata'
+        path: 'fluxengine-testdata'
     - name: brew
-      run: brew install sqlite pkg-config libusb protobuf wxwidgets fmt make coreutils
+      run: |
+        brew install sqlite pkg-config libusb protobuf wxwidgets fmt make coreutils dylibbundler libjpeg
     - name: make
-      run: gmake
+      run: gmake -C fluxengine
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ github.event.repository.name }}.${{ github.sha }}.fluxengine.${{ runner.arch }}.pkg
+        path: |
+          fluxengine/FluxEngine.pkg
+          fluxengine/FluxEngine.app.zip
 
   build-windows:
     runs-on: windows-latest
-    defaults:
-      run:
-        shell: msys2 {0}
+
     steps:
-    - uses: msys2/setup-msys2@v2
+    - name: setup WSL
+      run: |
+        curl -L https://github.com/WhitewaterFoundry/Fedora-Remix-for-WSL/releases/download/41.0.0/Fedora-Remix-for-WSL-SL_41.0.0.0_x64_arm64.msixbundle -o fedora.msixbundle
+        unzip fedora.msixbundle Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix
+        unzip Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix install.tar.gz
+        wsl --update
+        wsl --set-default-version 1
+        wsl --import fedora fedora install.tar.gz
+        wsl --set-default fedora
+        wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
+        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico ninja-build'
+
+    - name: fix line endings
+      run: |
+        git config --global core.autocrlf false
+        git config --global core.eol lf
+        
+    - uses: actions/checkout@v4
       with:
-        update: true
-        msystem: MINGW32
-        install: >-
-          diffutils
-          make
-          mingw-w64-i686-fmt
-          mingw-w64-i686-gcc
-          mingw-w64-i686-libusb
-          mingw-w64-i686-pkg-config
-          mingw-w64-i686-protobuf
-          mingw-w64-i686-sqlite3
-          mingw-w64-i686-wxWidgets
-          mingw-w64-i686-zlib
-          mingw-w64-i686-nsis
-          zip
-    - uses: actions/checkout@v1
-    - name: build
-      run: make
+        repository: 'davidgiven/fluxengine'
+        path: 'fluxengine'
+
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine-testdata'
+        path: 'fluxengine-testdata'
+
+    - name: run
+      run: |
+        wsl sh -c 'make -C fluxengine BUILDTYPE=windows -j$(nproc)'
 
     - name: nsis
       run: |
-        strip fluxengine.exe -o fluxengine-stripped.exe
-        strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe
-        makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi
+        wsl sh -c 'cd fluxengine && strip fluxengine.exe -o fluxengine-stripped.exe'
+        wsl sh -c 'cd fluxengine && strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe'
+        wsl sh -c 'cd fluxengine && makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi'
 
     - name: zip
       run: |
-        zip -9 fluxengine.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex fluxengine-installer.exe
+        wsl sh -c 'cd fluxengine && zip -9 fluxengine-windows.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex fluxengine-installer.exe'
 
     - name: Upload build artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: ${{ github.event.repository.name }}.${{ github.sha }}
-        path: fluxengine.zip
+        name: ${{ github.event.repository.name }}.${{ github.sha }}.windows.zip
+        path: fluxengine/fluxengine-windows.zip

.github/workflows/release.yml

@@ -1,5 +1,9 @@
 name: Autorelease
 
+concurrency: 
+  group: environment-release-${{ github.head_ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:
@@ -8,42 +12,43 @@ on:
 jobs:
   dev-release:
     runs-on: windows-latest
-    defaults:
-      run:
-        shell: msys2 {0}
-    steps:
-    - uses: msys2/setup-msys2@v2
-      with:
-        update: true
-        msystem: MINGW32
-        install: >-
-          diffutils
-          make
-          mingw-w64-i686-fmt
-          mingw-w64-i686-gcc
-          mingw-w64-i686-libusb
-          mingw-w64-i686-pkg-config
-          mingw-w64-i686-protobuf
-          mingw-w64-i686-sqlite3
-          mingw-w64-i686-wxWidgets
-          mingw-w64-i686-zlib
-          mingw-w64-i686-nsis
-          zip
-    - uses: actions/checkout@v3
 
-    - name: build
+    steps:
+    - name: setup WSL
       run: |
-        make
+        curl -L https://github.com/WhitewaterFoundry/Fedora-Remix-for-WSL/releases/download/41.0.0/Fedora-Remix-for-WSL-SL_41.0.0.0_x64_arm64.msixbundle -o fedora.msixbundle
+        unzip fedora.msixbundle Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix
+        unzip Fedora-Remix-for-WSL-SL_41.0.0.0_x64.msix install.tar.gz
+        wsl --update
+        wsl --set-default-version 1
+        wsl --import fedora fedora install.tar.gz
+        wsl --set-default fedora
+        wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
+        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico ninja-build'
+
+    - name: fix line endings
+      run: |
+        git config --global core.autocrlf false
+        git config --global core.eol lf
+        
+    - uses: actions/checkout@v4
+      with:
+        repository: 'davidgiven/fluxengine'
+        path: 'fluxengine'
+
+    - name: run
+      run: |
+        wsl sh -c 'cd fluxengine && make BUILDTYPE=windows -j$(nproc)'
 
     - name: nsis
       run: |
-        strip fluxengine.exe -o fluxengine-stripped.exe
-        strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe
-        makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi
+        wsl sh -c 'cd fluxengine && strip fluxengine.exe -o fluxengine-stripped.exe'
+        wsl sh -c 'cd fluxengine && strip fluxengine-gui.exe -o fluxengine-gui-stripped.exe'
+        wsl sh -c 'cd fluxengine && makensis -v2 -nocd -dOUTFILE=fluxengine-installer.exe extras/windows-installer.nsi'
 
     - name: zip
       run: |
-        zip -9 fluxengine.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex
+        wsl sh -c 'cd fluxengine && zip -9 fluxengine-windows.zip fluxengine.exe fluxengine-gui.exe upgrade-flux-file.exe brother120tool.exe brother240tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex fluxengine-installer.exe'
 
     - name: date
       run: |
@@ -54,6 +59,7 @@ jobs:
       with:
         tag-name: dev
         force-branch: false
+        git-directory: 'fluxengine'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -72,8 +78,57 @@ jobs:
       with:
         name: Development build ${{ env.RELEASE_DATE }}
         files: |
-          fluxengine.zip
-          fluxengine-installer.exe
+          fluxengine/fluxengine.zip
+          fluxengine/fluxengine-installer.exe
         tag_name: dev
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  build-macos:
+    strategy:
+      matrix:
+        runs-on: [macos-13, macos-latest]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: brew
+      run: brew install sqlite pkg-config libusb protobuf wxwidgets fmt make coreutils dylibbundler libjpeg
+
+    - name: make
+      run: |
+        gmake
+        mv FluxEngine.pkg FluxEngine-${{ runner.arch }}.pkg
+        mv FluxEngine.app.zip FluxEngine-${{ runner.arch }}.app.zip
+
+    - name: tag
+      uses: EndBug/latest-tag@latest
+      with:
+        tag-name: dev
+        force-branch: false
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: delete-old-assets
+      uses: mknejp/delete-release-assets@v1
+      with:
+        token: ${{ github.token }}
+        tag: dev
+        assets: | 
+          FluxEngine-${{ runner.arch }}.pkg
+          FluxEngine-${{ runner.arch }}.app.zip
+        fail-if-no-assets: false
+
+    - name: release
+      uses: softprops/action-gh-release@v1
+      with:
+        name: Development build ${{ env.RELEASE_DATE }}
+        files: |
+          FluxEngine-${{ runner.arch }}.pkg
+          FluxEngine-${{ runner.arch }}.app.zip
+        tag_name: dev
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+

.hgignore

@@ -1,4 +1,5 @@
 .obj
+.git
 streams
 .*\.flux
 .*\.img

Makefile

@@ -1,252 +1,116 @@
+ifeq ($(BUILDTYPE),)
+    buildtype_Darwin = osx
+    buildtype_Haiku = haiku
+    BUILDTYPE := $(buildtype_$(shell uname -s ))
+        ifeq ($(BUILDTYPE),)
+                BUILDTYPE := unix
+        endif
+endif
+export BUILDTYPE
+
+ifeq ($(BUILDTYPE),windows)
+	MINGW = i686-w64-mingw32-
+	CC = $(MINGW)gcc
+	CXX = $(MINGW)g++ -std=c++20
+	CFLAGS += -g -O3 \
+		-Wno-unknown-warning-option \
+		-ffunction-sections \
+		-fdata-sections
+	CXXFLAGS += \
+		-fext-numeric-literals \
+		-Wno-deprecated-enum-float-conversion \
+		-Wno-deprecated-enum-enum-conversion
+	LDFLAGS += -static -Wl,--gc-sections
+	AR = $(MINGW)ar
+	PKG_CONFIG = $(MINGW)pkg-config -static
+	WINDRES = $(MINGW)windres
+	WX_CONFIG = /usr/i686-w64-mingw32/sys-root/mingw/bin/wx-config-3.0 --static=yes
+	EXT = .exe
+else
+	CC = gcc
+	CXX = g++ -std=c++20
+	CFLAGS = -g -O3 \
+		-Wno-unknown-warning-option
+	CXXFLAGS += \
+		-Wno-deprecated-enum-float-conversion \
+		-Wno-deprecated-enum-enum-conversion
+	LDFLAGS =
+	AR = ar
+	PKG_CONFIG = pkg-config
+	ifeq ($(BUILDTYPE),osx)
+	else
+		LDFLAGS += -pthread -Wl,--no-as-needed
+	endif
+endif
+
+HOSTCC = gcc
+HOSTCXX = g++ -std=c++20
+HOSTCFLAGS = -g -O3
+HOSTLDFLAGS =
+
+REALOBJ = .obj
+OBJ = $(REALOBJ)/$(BUILDTYPE)
+DESTDIR ?=
+PREFIX ?= /usr/local
+BINDIR ?= $(PREFIX)/bin
+
 # Special Windows settings.
 
 ifeq ($(OS), Windows_NT)
+	EXT ?= .exe
 	MINGWBIN = /mingw32/bin
 	CCPREFIX = $(MINGWBIN)/
-	LUA = $(MINGWBIN)/lua
 	PKG_CONFIG = $(MINGWBIN)/pkg-config
 	WX_CONFIG = /usr/bin/sh $(MINGWBIN)/wx-config --static=yes
 	PROTOC = $(MINGWBIN)/protoc
-	PLATFORM = WINDOWS
+	WINDRES = windres
 	LDFLAGS += \
 		-static
 	CXXFLAGS += \
-		-std=c++17 \
 		-fext-numeric-literals \
 		-Wno-deprecated-enum-float-conversion \
 		-Wno-deprecated-enum-enum-conversion
 
-	# Required to get the gcc run-time libraries on the path.
+	# Required to get the gcc run - time libraries on the path.
 	export PATH := $(PATH):$(MINGWBIN)
-	EXT ?= .exe
 endif
 
 # Special OSX settings.
 
 ifeq ($(shell uname),Darwin)
-	PLATFORM = OSX
 	LDFLAGS += \
 		-framework IOKit \
-		-framework Foundation
+		-framework Foundation 
 endif
 
-# Check the Make version.
+.PHONY: all
+all: +all README.md
 
+.PHONY: binaries tests
+binaries: all
+tests: all
+	
+README.md: $(OBJ)/scripts/+mkdocindex/mkdocindex$(EXT)
+	@echo $(PROGRESSINFO)MKDOC $@
+	@csplit -s -f$(OBJ)/README. README.md '/<!-- FORMATSSTART -->/' '%<!-- FORMATSEND -->%'
+	@(cat $(OBJ)/README.00 && $< && cat $(OBJ)/README.01) > README.md
 
-ifeq ($(findstring 4.,$(MAKE_VERSION)),)
-$(error You need GNU Make 4.x for this (if you're on OSX, use gmake).)
-endif
+.PHONY: tests
 
-# Normal settings.
+clean::
+	$(hide) rm -rf $(REALOBJ)
 
-OBJDIR ?= .obj
-CCPREFIX ?=
-LUA ?= lua
-CC ?= $(CCPREFIX)gcc
-CXX ?= $(CCPREFIX)g++
-AR ?= $(CCPREFIX)ar
-PKG_CONFIG ?= pkg-config
-WX_CONFIG ?= wx-config
-PROTOC ?= protoc
-CFLAGS ?= -g -O3
-CXXFLAGS += -std=c++17
-LDFLAGS ?=
-PLATFORM ?= UNIX
-TESTS ?= yes
-EXT ?=
-DESTDIR ?=
-PREFIX ?= /usr/local
-BINDIR ?= $(PREFIX)/bin
+include build/ab.mk
 
-CFLAGS += \
-	-Iarch \
-	-Ilib \
-	-I. \
-	-I$(OBJDIR)/arch \
-	-I$(OBJDIR)/lib \
-	-I$(OBJDIR) \
+DOCKERFILES = \
+	debian11 \
+    debian12 \
+    fedora40 \
+    fedora41 \
+	manjaro
 
-LDFLAGS += \
-	-lz \
-	-lfmt
+docker-%: tests/docker/Dockerfile.%
+	docker build -t $* -f $< .
 
-.SUFFIXES:
-.DELETE_ON_ERROR:
-
-define nl
-
-endef
-
-use-library = $(eval $(use-library-impl))
-define use-library-impl
-$1: $(call $3_LIB)
-$1: private LDFLAGS += $(call $3_LDFLAGS)
-$2: private CFLAGS += $(call $3_CFLAGS)
-endef
-
-use-pkgconfig = $(eval $(use-pkgconfig-impl))
-define use-pkgconfig-impl
-ifneq ($(strip $(shell $(PKG_CONFIG) $3; echo $$?)),0)
-$$(error Missing required pkg-config dependency: $3)
-endif
-
-$(1): private LDFLAGS += $(shell $(PKG_CONFIG) --libs $(3))
-$(2): private CFLAGS += $(shell $(PKG_CONFIG) --cflags $(3))
-endef
-
-.PHONY: all binaries tests clean install install-bin
-all: binaries tests
-
-PROTOS = \
-	arch/aeslanier/aeslanier.proto \
-	arch/agat/agat.proto \
-	arch/amiga/amiga.proto \
-	arch/apple2/apple2.proto \
-	arch/brother/brother.proto \
-	arch/c64/c64.proto \
-	arch/f85/f85.proto \
-	arch/fb100/fb100.proto \
-	arch/ibm/ibm.proto \
-	arch/macintosh/macintosh.proto \
-	arch/micropolis/micropolis.proto \
-	arch/mx/mx.proto \
-	arch/northstar/northstar.proto \
-	arch/tids990/tids990.proto \
-	arch/victor9k/victor9k.proto \
-	arch/zilogmcz/zilogmcz.proto \
-	lib/common.proto \
-	lib/config.proto \
-	lib/decoders/decoders.proto \
-	lib/drive.proto \
-	lib/encoders/encoders.proto \
-	lib/fl2.proto \
-	lib/fluxsink/fluxsink.proto \
-	lib/fluxsource/fluxsource.proto \
-	lib/imagereader/imagereader.proto \
-	lib/imagewriter/imagewriter.proto \
-	lib/layout.proto \
-	lib/usb/usb.proto \
-	lib/vfs/vfs.proto \
-	tests/testproto.proto \
-
-PROTO_HDRS = $(patsubst %.proto, $(OBJDIR)/%.pb.h, $(PROTOS))
-PROTO_SRCS = $(patsubst %.proto, $(OBJDIR)/%.pb.cc, $(PROTOS))
-PROTO_OBJS = $(patsubst %.cc, %.o, $(PROTO_SRCS))
-PROTO_CFLAGS = $(shell $(PKG_CONFIG) --cflags protobuf)
-$(PROTO_SRCS): | $(PROTO_HDRS)
-$(PROTO_OBJS): CFLAGS += $(PROTO_CFLAGS)
-PROTO_LIB = $(OBJDIR)/libproto.a
-$(PROTO_LIB): $(PROTO_OBJS)
-PROTO_LDFLAGS = $(shell $(PKG_CONFIG) --libs protobuf) -pthread $(PROTO_LIB)
-.PRECIOUS: $(PROTO_HDRS) $(PROTO_SRCS)
-
-include dep/agg/build.mk
-include dep/libusbp/build.mk
-include dep/stb/build.mk
-include dep/emu/build.mk
-include dep/fatfs/build.mk
-include dep/adflib/build.mk
-include dep/hfsutils/build.mk
-include scripts/build.mk
-
-include lib/build.mk
-include arch/build.mk
-include src/build.mk
-include src/gui/build.mk
-include tools/build.mk
-include tests/build.mk
-
-do-encodedecodetest = $(eval $(do-encodedecodetest-impl))
-define do-encodedecodetest-impl
-
-tests: $(OBJDIR)/$1$3.encodedecode
-$(OBJDIR)/$1$3.encodedecode: scripts/encodedecodetest.sh $(FLUXENGINE_BIN) $2
-	@mkdir -p $(dir $$@)
-	@echo ENCODEDECODETEST $1 $3
-	@scripts/encodedecodetest.sh $1 flux $(FLUXENGINE_BIN) $2 $3 > $$@
-
-endef
-
-$(call do-encodedecodetest,amiga)
-$(call do-encodedecodetest,apple2)
-$(call do-encodedecodetest,atarist360)
-$(call do-encodedecodetest,atarist370)
-$(call do-encodedecodetest,atarist400)
-$(call do-encodedecodetest,atarist410)
-$(call do-encodedecodetest,atarist720)
-$(call do-encodedecodetest,atarist740)
-$(call do-encodedecodetest,atarist800)
-$(call do-encodedecodetest,atarist820)
-$(call do-encodedecodetest,bk800)
-$(call do-encodedecodetest,brother120)
-$(call do-encodedecodetest,brother240)
-$(call do-encodedecodetest,commodore1541,scripts/commodore1541_test.textpb,--35)
-$(call do-encodedecodetest,commodore1541,scripts/commodore1541_test.textpb,--40)
-$(call do-encodedecodetest,commodore1581)
-$(call do-encodedecodetest,cmd_fd2000)
-$(call do-encodedecodetest,hp9121)
-$(call do-encodedecodetest,ibm1200)
-$(call do-encodedecodetest,ibm1232)
-$(call do-encodedecodetest,ibm1440)
-$(call do-encodedecodetest,ibm180)
-$(call do-encodedecodetest,ibm360)
-$(call do-encodedecodetest,ibm720)
-$(call do-encodedecodetest,mac400,scripts/mac400_test.textpb)
-$(call do-encodedecodetest,mac800,scripts/mac800_test.textpb)
-$(call do-encodedecodetest,n88basic)
-$(call do-encodedecodetest,rx50)
-$(call do-encodedecodetest,tids990)
-$(call do-encodedecodetest,trs80_88,,--35)
-$(call do-encodedecodetest,trs80_88,,--40)
-$(call do-encodedecodetest,trs80_175,,--35)
-$(call do-encodedecodetest,trs80_175,,--40)
-$(call do-encodedecodetest,victor9k_ss)
-$(call do-encodedecodetest,victor9k_ds)
-
-$(OBJDIR)/%.a:
-	@mkdir -p $(dir $@)
-	@echo AR $@
-	@$(AR) rc $@ $^
-
-%.exe:
-	@mkdir -p $(dir $@)
-	@echo LINK $@
-	@$(CXX) -o $@ $^ $(LDFLAGS) $(LDFLAGS)
-
-$(OBJDIR)/%.o: %.cpp
-	@mkdir -p $(dir $@)
-	@echo CXX $<
-	@$(CXX) $(CFLAGS) $(CXXFLAGS) -MMD -MP -MF $(@:.o=.d) -c -o $@ $<
-
-$(OBJDIR)/%.o: %.cc
-	@mkdir -p $(dir $@)
-	@echo CXX $<
-	@$(CXX) $(CFLAGS) $(CXXFLAGS) -MMD -MP -MF $(@:.o=.d) -c -o $@ $<
-
-$(OBJDIR)/%.o: $(OBJDIR)/%.cc
-	@mkdir -p $(dir $@)
-	@echo CXX $<
-	@$(CXX) $(CFLAGS) $(CXXFLAGS) -MMD -MP -MF $(@:.o=.d) -c -o $@ $<
-
-$(OBJDIR)/%.o: %.c
-	@mkdir -p $(dir $@)
-	@echo CC $<
-	@$(CC) $(CFLAGS) $(CFLAGS) -MMD -MP -MF $(@:.o=.d) -c -o $@ $<
-
-$(OBJDIR)/%.pb.h: %.proto
-	@mkdir -p $(dir $@)
-	@echo PROTOC $@
-	@$(PROTOC) -I. --cpp_out=$(OBJDIR) $<
-
-clean:
-	rm -rf $(OBJDIR)
-
-install: install-bin # install-man install-docs ...
-
-install-bin: fluxengine$(EXT) fluxengine-gui$(EXT) brother120tool$(EXT) brother240tool$(EXT) upgrade-flux-file$(EXT)
-	install -d "$(DESTDIR)$(BINDIR)"
-	for target in $^; do \
-		install $$target "$(DESTDIR)$(BINDIR)/$$target"; \
-	done
-
--include $(OBJS:%.o=%.d)
+.PHONY: dockertests
+dockertests: $(foreach f,$(DOCKERFILES), docker-$(strip $f) .WAIT)

README.md

@@ -4,11 +4,8 @@ FluxEngine
 (If you're reading this on GitHub, the formatting's a bit messed up. [Try the
 version on cowlark.com instead.](http://cowlark.com/fluxengine/))
 
-**Breaking news!** As of 2022-09-09, there's new [filesystem
-support](doc/filesystem.md). Read (and sometimes write) files directly from
-(and to) your disks, with eight different file systems! It works in the GUI,
-too, which is available for Linux (and other Unix clones), Windows and OSX. See
-the details below.
+**Breaking news!** As of 2024-10-01, the FluxEngine client software works
+(to a point) with [Applesauce](doc/applesauce.md) hardware.
 
 <div style="text-align: center">
 <a href="doc/screenshot.jpg"><img src="doc/screenshot.jpg" style="width:60%" alt="screenshot of the GUI in action"></a>
@@ -35,12 +32,14 @@ Don't believe me? Watch the demo reel!
 </div>
 
 **New!** The FluxEngine client software now works with
-[GreaseWeazle](https://github.com/keirf/Greaseweazle/wiki) hardware. So, if you
-can't find a PSoC5 development kit, or don't want to use the Cypress Windows
-tools for programming it, you can use one of these instead. Very nearly all
-FluxEngine features are available with the GreaseWeazle and it works out-of-the
-box. See the [dedicated GreaseWeazle documentation page](doc/greaseweazle.md)
-for more information.
+[Greaseweazle](https://github.com/keirf/Greaseweazle/wiki) and
+[Applesauce](https://applesaucefdc.com/) hardware. So, if you can't find a PSoC5
+development kit, or don't want to use the Cypress Windows tools for programming
+it, you can use one of these instead. Very nearly all FluxEngine features are
+available with the Greaseweazle and it works out-of-the box; the Applesauce is a
+bit less supported but still works. See the [dedicated Greaseweazle
+documentation page](doc/greaseweazle.md) or the [Applesauce
+page](doc/applesauce.md) for more information.
 
 Where?
 ------
@@ -65,7 +64,7 @@ following friendly articles:
   - [Using a FluxEngine](doc/using.md) ∾ what to do with your new hardware ∾
     flux files and image files ∾ knowing what you're doing
 
-  - [Using GreaseWeazle hardware with the FluxEngine client
+  - [Using Greaseweazle hardware with the FluxEngine client
     software](doc/greaseweazle.md) ∾ what works ∾ what doesn't work ∾ where to
     go for help
 
@@ -88,61 +87,64 @@ Which?
 
 The current support state is as follows.
 
-Dinosaurs (🦖) have yet to be observed in real life --- I've written the
-decoder based on Kryoflux (or other) dumps I've found. I don't (yet) have
-real, physical disks in my hand to test the capture process.
+Dinosaurs (🦖) have yet to be observed in real life --- I've written the encoder
+and/or decoder based on Kryoflux (or other) dumps I've found. I don't (yet) have
+real, physical disks in my hand to test the capture process, or hardware to
+verify that written disks work.
 
-Unicorns (🦄) are completely real --- this means that I've read actual,
-physical disks with these formats and so know they work (or had reports from
-people who've had it work).
+Unicorns (🦄) are completely real --- this means that I've read actual, physical
+disks with these formats and/or written real, physical disks and then used them
+on real hardware, and so know they work (or had reports from people who've had
+it work).
 
-### Old disk formats
+If a filesystem is listed, this means that FluxEngine natively supports that
+particular filesystem and can read (and sometimes write, support varies) files
+directly from disks, flux files or disk images. Some formats have multiple
+choices because they can store multiple types of file system.
 
-| Format                                    | Read? | Write? | Notes |
-|:------------------------------------------|:-----:|:------:|-------|
-| [IBM PC compatible](doc/disk-ibm.md)      |  🦄   |   🦄   | and compatibles (like the Atari ST) |
-| [Atari ST](doc/disk-atarist.md)           |  🦄   |   🦄   | technically the same as IBM, almost |
-| [Acorn ADFS](doc/disk-acornadfs.md)       |  🦄   |   🦖*  | single- and double- sided           |
-| [Acorn DFS](doc/disk-acorndfs.md)         |  🦄   |   🦖*  |                                     |
-| [Ampro Little Board](doc/disk-ampro.md)   |  🦖   |   🦖*  |                                     |
-| [Agat](doc/disk-agat.md)                  |  🦖   |        | Soviet Union Apple-II-like computer |
-| [Apple II](doc/disk-apple2.md)            |  🦄   |   🦄   |                                     |
-| [Amiga](doc/disk-amiga.md)                |  🦄   |   🦄   |                                     |
-| [Commodore 64 1541/1581](doc/disk-c64.md) |  🦄   |   🦄   | and probably the other formats      |
-| [Brother 120kB](doc/disk-brother.md)      |  🦄   |   🦄   |                                     |
-| [Brother 240kB](doc/disk-brother.md)      |  🦄   |   🦄   |                                     |
-| [Brother FB-100](doc/disk-fb100.md)       |  🦖   |        | Tandy Model 100, Husky Hunter, knitting machines |
-| [Elektronika BK](doc/disk-bd.md)          |  🦄   |   🦄   | Soviet Union PDP-11 clone           |
-| [Macintosh 400kB/800kB](doc/disk-macintosh.md)  |  🦄   |   🦄   |                                     |
-| [NEC PC-98](doc/disk-ibm.md)              |  🦄   |   🦄   | trimode drive not required          |
-| [Sharp X68000](doc/disk-ibm.md)           |  🦄   |   🦄   |                                     |
-| [TRS-80](doc/disk-trs80.md)               |  🦖   |   🦖*  | a minor variation of the IBM scheme |
+<!-- FORMATSSTART -->
+<!-- This section is automatically generated. Do not edit. -->
+
+| Profile | Format | Read? | Write? | Filesystem? |
+|:--------|:-------|:-----:|:------:|:------------|
+| [`acornadfs`](doc/disk-acornadfs.md) | Acorn ADFS: BBC Micro, Archimedes | 🦖 |  |  |
+| [`acorndfs`](doc/disk-acorndfs.md) | Acorn DFS: Acorn Atom, BBC Micro series | 🦄 |  | ACORNDFS  |
+| [`aeslanier`](doc/disk-aeslanier.md) | AES Lanier "No Problem": 616kB 5.25" 77-track SSDD hard sectored | 🦖 |  |  |
+| [`agat`](doc/disk-agat.md) | Agat: 840kB 5.25" 80-track DS | 🦖 | 🦖 |  |
+| [`amiga`](doc/disk-amiga.md) | Amiga: 880kB 3.5" DSDD | 🦄 | 🦄 | AMIGAFFS  |
+| [`ampro`](doc/disk-ampro.md) | Ampro Little Board: CP/M | 🦖 |  | CPMFS  |
+| [`apple2`](doc/disk-apple2.md) | Apple II: Prodos, Appledos, and CP/M | 🦄 | 🦄 | APPLEDOS CPMFS PRODOS  |
+| [`atarist`](doc/disk-atarist.md) | Atari ST: Almost PC compatible | 🦄 | 🦄 |  |
+| [`bk`](doc/disk-bk.md) | BK: 800kB 5.25"/3.5" 80-track 10-sector DSDD | 🦖 | 🦖 |  |
+| [`brother`](doc/disk-brother.md) | Brother word processors: GCR family | 🦄 | 🦄 | BROTHER120 FATFS  |
+| [`commodore`](doc/disk-commodore.md) | Commodore: 1541, 1581, 8050 and variations | 🦄 | 🦄 | CBMFS  |
+| [`eco1`](doc/disk-eco1.md) | VDS Eco1: CP/M; 1210kB 77-track mixed format DSHD | 🦖 |  | CPMFS  |
+| [`epsonpf10`](doc/disk-epsonpf10.md) | Epson PF-10: CP/M; 3.5" 40-track DSDD | 🦖 |  | CPMFS  |
+| [`f85`](doc/disk-f85.md) | Durango F85: 461kB 5.25" 77-track SS | 🦖 |  |  |
+| [`fb100`](doc/disk-fb100.md) | Brother FB-100: 100kB 3.5" 40-track SSSD | 🦖 |  |  |
+| [`hplif`](doc/disk-hplif.md) | Hewlett-Packard LIF: a variety of disk formats used by HP | 🦄 | 🦄 | LIF  |
+| [`ibm`](doc/disk-ibm.md) | IBM PC: Generic PC 3.5"/5.25" disks | 🦄 | 🦄 | FATFS  |
+| [`icl30`](doc/disk-icl30.md) | ICL Model 30: CP/M; 263kB 35-track DSSD | 🦖 |  | CPMFS  |
+| [`juku`](doc/disk-juku.md) | Juku E5104: CP/M |  |  | CPMFS  |
+| [`mac`](doc/disk-mac.md) | Macintosh: 400kB/800kB 3.5" GCR | 🦄 | 🦄 | MACHFS  |
+| [`micropolis`](doc/disk-micropolis.md) | Micropolis: 100tpi MetaFloppy disks | 🦄 | 🦄 |  |
+| [`ms2000`](doc/disk-ms2000.md) | : MS2000 Microdisk Development System |  |  | MICRODOS  |
+| [`mx`](doc/disk-mx.md) | DVK MX: Soviet-era PDP-11 clone | 🦖 |  |  |
+| [`n88basic`](doc/disk-n88basic.md) | N88-BASIC: PC8800/PC98 5.25" 77-track 26-sector DSHD | 🦄 | 🦄 |  |
+| [`northstar`](doc/disk-northstar.md) | Northstar: 5.25" hard sectored | 🦄 | 🦄 |  |
+| [`psos`](doc/disk-psos.md) | pSOS: 800kB DSDD with PHILE | 🦄 | 🦄 | PHILE  |
+| [`rolandd20`](doc/disk-rolandd20.md) | Roland D20: 3.5" electronic synthesiser disks | 🦄 | 🦖 | ROLAND  |
+| [`rx50`](doc/disk-rx50.md) | Digital RX50: 400kB 5.25" 80-track 10-sector SSDD | 🦖 | 🦖 |  |
+| [`smaky6`](doc/disk-smaky6.md) | Smaky 6: 308kB 5.25" 77-track 16-sector SSDD, hard sectored | 🦖 |  | SMAKY6  |
+| [`tartu`](doc/disk-tartu.md) | Tartu: The Palivere and variations | 🦄 | 🦖 | CPMFS  |
+| [`ti99`](doc/disk-ti99.md) | TI-99: 90kB 35-track SSSD | 🦖 |  |  |
+| [`tids990`](doc/disk-tids990.md) | Texas Instruments DS990: 1126kB 8" DSSD | 🦖 | 🦖 |  |
+| [`tiki`](doc/disk-tiki.md) | Tiki 100: CP/M |  |  | CPMFS  |
+| [`victor9k`](doc/disk-victor9k.md) | Victor 9000 / Sirius One: 1224kB 5.25" DSDD GCR | 🦖 | 🦖 |  |
+| [`zilogmcz`](doc/disk-zilogmcz.md) | Zilog MCZ: 320kB 8" 77-track SSSD hard-sectored | 🦖 |  | ZDOS  |
 {: .datatable }
 
-`*`: these formats are variations of the generic IBM format, and since the
-IBM writer is completely generic, it should be configurable for these
-formats... theoretically. I don't have the hardware to try it.
-
-### Even older disk formats
-
-These formats are for particularly old, weird architectures, even by the
-standards of floppy disks. They've largely been implemented from single flux
-files with no access to physical hardware. Typically the reads were pretty
-bad and I've had to make a number of guesses as to how things work. They do,
-at least, check the CRC so what data's there is probably good.
-
-| Format                                   | Read? | Write? | Notes |
-|:-----------------------------------------|:-----:|:------:|-------|
-| [AES Superplus / No Problem](doc/disk-aeslanier.md) |  🦖   | | hard sectors! |
-| [Durango F85](doc/disk-durangof85.md)    |  🦖   |        | 5.25" |
-| [DVK MX](doc/disk-mx.md)                 |  🦖   |        | Soviet PDP-11 clone |
-| [VDS Eco1](doc/disk-eco1.md)             |  🦖   |        | 8" mixed format |
-| [Micropolis](doc/disk-micropolis.md)     |  🦄   |        | Micropolis 100tpi drives |
-| [Northstar](doc/disk-northstar.md)       |  🦖   |   🦖   | 5.25" hard sectors |
-| [TI DS990 FD1000](doc/disk-tids990.md)   |  🦄   |  🦄    | 8" |
-| [Victor 9000](doc/disk-victor9k.md)      |  🦖   |        | 5.25" GCR encoded |
-| [Zilog MCZ](doc/disk-zilogmcz.md)        |  🦖   |        | 8" _and_ hard sectors |
-{: .datatable }
+<!-- FORMATSEND -->
 
 ### Notes
 
@@ -257,9 +259,16 @@ package, written by Robert Leslie et al, taken from
 https://www.mars.org/home/rob/proj/hfs. It is GPL 2.0 licensed. Please see the
 contents of the directory for the full text.
 
+As an exception, `dep/lexy` contains a partial copy of the lexy package, written
+by foonathen@github, taken from https://github.com/foonathan/lexy. It is BSL 1.0
+licensed. Please see the contents of the directory for the full text.
+
+As an exception, `dep/alphanum` contains a copy of the alphanum package,
+written by Dave Koelle, taken from
+https://web.archive.org/web/20210207124255/davekoelle.com/alphanum.html. It is
+MIT licensed. Please see the source for the full text.
+
 __Important:__ Because of all these exceptions, if you distribute the
 FluxEngine package as a whole, you must comply with the terms of _all_ of the
 licensing terms. This means that __effectively the FluxEngine package is
 distributable under the terms of the GPL 2.0__.
-
-

arch/aeslanier/aeslanier.h

@@ -2,9 +2,10 @@
 #define AESLANIER_H
 
 #define AESLANIER_RECORD_SEPARATOR 0x55555122
-#define AESLANIER_SECTOR_LENGTH    256
-#define AESLANIER_RECORD_SIZE      (AESLANIER_SECTOR_LENGTH + 5)
+#define AESLANIER_SECTOR_LENGTH 256
+#define AESLANIER_RECORD_SIZE (AESLANIER_SECTOR_LENGTH + 5)
 
-extern std::unique_ptr<Decoder> createAesLanierDecoder(const DecoderProto& config);
+extern std::unique_ptr<Decoder> createAesLanierDecoder(
+    const DecoderProto& config);
 
 #endif

arch/aeslanier/decoder.cc

@@ -1,76 +1,65 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "aeslanier.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "sector.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/aeslanier/aeslanier.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 
 static const FluxPattern SECTOR_PATTERN(32, AESLANIER_RECORD_SEPARATOR);
 
-/* This is actually M2FM, rather than MFM, but it our MFM/FM decoder copes fine with it. */
-
-static Bytes reverse_bits(const Bytes& input)
-{
-    Bytes output;
-    ByteWriter bw(output);
-
-    for (uint8_t b : input)
-        bw.write_8(reverse_bits(b));
-    return output;
-}
+/* This is actually M2FM, rather than MFM, but it our MFM/FM decoder copes fine
+ * with it. */
 
 class AesLanierDecoder : public Decoder
 {
 public:
-	AesLanierDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    AesLanierDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(SECTOR_PATTERN);
-	}
+    {
+        return seekToPattern(SECTOR_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		/* Skip ID mark (we know it's a AESLANIER_RECORD_SEPARATOR). */
+    {
+        /* Skip ID mark (we know it's a AESLANIER_RECORD_SEPARATOR). */
 
-		readRawBits(16);
+        readRawBits(16);
 
-		const auto& rawbits = readRawBits(AESLANIER_RECORD_SIZE*16);
-		const auto& bytes = decodeFmMfm(rawbits).slice(0, AESLANIER_RECORD_SIZE);
-		const auto& reversed = reverse_bits(bytes);
+        const auto& rawbits = readRawBits(AESLANIER_RECORD_SIZE * 16);
+        const auto& bytes =
+            decodeFmMfm(rawbits).slice(0, AESLANIER_RECORD_SIZE);
+        const auto& reversed = bytes.reverseBits();
 
-		_sector->logicalTrack = reversed[1];
-		_sector->logicalSide = 0;
-		_sector->logicalSector = reversed[2];
+        _sector->logicalTrack = reversed[1];
+        _sector->logicalSide = 0;
+        _sector->logicalSector = reversed[2];
 
-		/* Check header 'checksum' (which seems far too simple to mean much). */
+        /* Check header 'checksum' (which seems far too simple to mean much). */
 
-		{
-			uint8_t wanted = reversed[3];
-			uint8_t got = reversed[1] + reversed[2];
-			if (wanted != got)
-				return;
-		}
+        {
+            uint8_t wanted = reversed[3];
+            uint8_t got = reversed[1] + reversed[2];
+            if (wanted != got)
+                return;
+        }
 
-		/* Check data checksum, which also includes the header and is
-			* significantly better. */
+        /* Check data checksum, which also includes the header and is
+         * significantly better. */
 
-		_sector->data = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
-		uint16_t wanted = reversed.reader().seek(0x101).read_le16();
-		uint16_t got = crc16ref(MODBUS_POLY_REF, _sector->data);
-		_sector->status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
+        uint16_t wanted = reversed.reader().seek(0x101).read_le16();
+        uint16_t got = crc16ref(MODBUS_POLY_REF, _sector->data);
+        _sector->status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createAesLanierDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new AesLanierDecoder(config));
+    return std::unique_ptr<Decoder>(new AesLanierDecoder(config));
 }
-
-

arch/agat/agat.cc

@@ -1,22 +1,20 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "agat.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/agat/agat.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 
 uint8_t agatChecksum(const Bytes& bytes)
 {
     uint16_t checksum = 0;
 
-	for (uint8_t b : bytes)
-	{
-		if (checksum > 0xff)
-			checksum = (checksum + 1) & 0xff;
+    for (uint8_t b : bytes)
+    {
+        if (checksum > 0xff)
+            checksum = (checksum + 1) & 0xff;
 
-		checksum += b;
-	}
+        checksum += b;
+    }
 
-	return checksum & 0xff;
+    return checksum & 0xff;
 }
-
-

arch/agat/agat.h

@@ -3,9 +3,17 @@
 
 #define AGAT_SECTOR_SIZE 256
 
+static constexpr uint64_t SECTOR_ID = 0x8924555549111444;
+static constexpr uint64_t DATA_ID = 0x8924555514444911;
+
+class Encoder;
+class EncoderProto;
+class Decoder;
+class DecoderProto;
+
 extern std::unique_ptr<Decoder> createAgatDecoder(const DecoderProto& config);
+extern std::unique_ptr<Encoder> createAgatEncoder(const EncoderProto& config);
 
 extern uint8_t agatChecksum(const Bytes& bytes);
 
 #endif
-

arch/agat/agat.proto

@@ -1,5 +1,19 @@
 syntax = "proto2";
 
+import "lib/config/common.proto";
+
 message AgatDecoderProto {}
 
+message AgatEncoderProto {
+	optional double target_clock_period_us = 1
+		[default=2.00, (help)="Data clock period of target format."];
+	optional double target_rotational_period_ms = 2
+		[default=200.0, (help)="Rotational period of target format."];
+	optional int32 post_index_gap_bytes = 3
+		[default=40, (help)="Post-index gap before first sector header."];
+	optional int32 pre_sector_gap_bytes = 4
+		[default=11, (help)="Gap before each sector header."];
+	optional int32 pre_data_gap_bytes = 5
+		[default=2, (help)="Gap before each sector data record."];
+}
 

arch/agat/decoder.cc

@@ -1,21 +1,23 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "agat.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "sector.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/agat/agat.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 
+// clang-format off
 /*
  * data:    X  X  X  X   X  X  X  X   X  -  -  X   -  X  -  X   -  X  X  -   X  -  X  -  = 0xff956a
  * flux:   01 01 01 01  01 01 01 01  01 00 10 01  00 01 00 01  00 01 01 00  01 00 01 00  = 0x555549111444
  * 
  * data:    X  X  X  X   X  X  X  X   -  X  X  -   X  -  X  -   X  -  -  X   -  X  -  X  = 0xff6a95
  * flux:   01 01 01 01  01 01 01 01  00 01 01 00  01 00 01 00  01 00 10 01  00 01 00 01  = 0x555514444911
- * 
+ *
  * Each pattern is prefixed with this one:
  *
  * data:          -  -   -  X   -  -   X  - = 0x12
@@ -30,68 +32,59 @@
  *          0100010010010010  = MFM encoded
  *           1000100100100100 = with trailing zero
  *            - - - X - - X - = effective bitstream = 0x12
- *
  */
+// clang-format on
 
-static const uint64_t SECTOR_ID = 0x8924555549111444;
 static const FluxPattern SECTOR_PATTERN(64, SECTOR_ID);
-
-static const uint64_t DATA_ID = 0x8924555514444911;
 static const FluxPattern DATA_PATTERN(64, DATA_ID);
 
-static const FluxMatchers ALL_PATTERNS = {
-	&SECTOR_PATTERN,
-	&DATA_PATTERN
-};
+static const FluxMatchers ALL_PATTERNS = {&SECTOR_PATTERN, &DATA_PATTERN};
 
 class AgatDecoder : public Decoder
 {
 public:
-	AgatDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    AgatDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ALL_PATTERNS);
-	}
+    {
+        return seekToPattern(ALL_PATTERNS);
+    }
 
     void decodeSectorRecord() override
-	{
-		if (readRaw64() != SECTOR_ID)
-			return;
+    {
+        if (readRaw64() != SECTOR_ID)
+            return;
 
-		auto bytes = decodeFmMfm(readRawBits(64)).slice(0, 4);
-		if (bytes[3] != 0x5a)
-			return;
+        auto bytes = decodeFmMfm(readRawBits(64)).slice(0, 4);
+        if (bytes[3] != 0x5a)
+            return;
 
-		_sector->logicalTrack = bytes[1] >> 1;
-		_sector->logicalSector = bytes[2];
-		_sector->logicalSide = bytes[1] & 1;
-		_sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
-	}
+        _sector->logicalTrack = bytes[1] >> 1;
+        _sector->logicalSector = bytes[2];
+        _sector->logicalSide = bytes[1] & 1;
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+    }
 
-	void decodeDataRecord() override
-	{
-		if (readRaw64() != DATA_ID)
-			return;
+    void decodeDataRecord() override
+    {
+        if (readRaw64() != DATA_ID)
+            return;
 
-		Bytes bytes = decodeFmMfm(readRawBits((AGAT_SECTOR_SIZE+2)*16)).slice(0, AGAT_SECTOR_SIZE+2);
+        Bytes bytes = decodeFmMfm(readRawBits((AGAT_SECTOR_SIZE + 2) * 16))
+                          .slice(0, AGAT_SECTOR_SIZE + 2);
 
-		if (bytes[AGAT_SECTOR_SIZE+1] != 0x5a)
-			return;
+        if (bytes[AGAT_SECTOR_SIZE + 1] != 0x5a)
+            return;
 
-		_sector->data = bytes.slice(0, AGAT_SECTOR_SIZE);
-		uint8_t wantChecksum = bytes[AGAT_SECTOR_SIZE];
-		uint8_t gotChecksum = agatChecksum(_sector->data);
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = bytes.slice(0, AGAT_SECTOR_SIZE);
+        uint8_t wantChecksum = bytes[AGAT_SECTOR_SIZE];
+        uint8_t gotChecksum = agatChecksum(_sector->data);
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createAgatDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new AgatDecoder(config));
+    return std::unique_ptr<Decoder>(new AgatDecoder(config));
 }
-
-
-

arch/agat/encoder.cc

@@ -0,0 +1,118 @@
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/agat/agat.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
+#include "lib/data/layout.h"
+#include "arch/agat/agat.pb.h"
+#include "lib/encoders/encoders.pb.h"
+
+class AgatEncoder : public Encoder
+{
+public:
+    AgatEncoder(const EncoderProto& config):
+        Encoder(config),
+        _config(config.agat())
+    {
+    }
+
+private:
+    void writeRawBits(uint64_t data, int width)
+    {
+        _cursor += width;
+        _lastBit = data & 1;
+        for (int i = 0; i < width; i++)
+        {
+            unsigned pos = _cursor - i - 1;
+            if (pos < _bits.size())
+                _bits[pos] = data & 1;
+            data >>= 1;
+        }
+    }
+
+    void writeBytes(const Bytes& bytes)
+    {
+        encodeMfm(_bits, _cursor, bytes, _lastBit);
+    }
+
+    void writeByte(uint8_t byte)
+    {
+        Bytes b;
+        b.writer().write_8(byte);
+        writeBytes(b);
+    }
+
+    void writeFillerRawBytes(int count, uint16_t byte)
+    {
+        for (int i = 0; i < count; i++)
+            writeRawBits(byte, 16);
+    };
+
+    void writeFillerBytes(int count, uint8_t byte)
+    {
+        Bytes b{byte};
+        for (int i = 0; i < count; i++)
+            writeBytes(b);
+    };
+
+public:
+    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+        const std::vector<std::shared_ptr<const Sector>>& sectors,
+        const Image& image) override
+    {
+        auto trackLayout = Layout::getLayoutOfTrack(
+            trackInfo->logicalTrack, trackInfo->logicalSide);
+
+        double clockRateUs = _config.target_clock_period_us() / 2.0;
+        int bitsPerRevolution =
+            (_config.target_rotational_period_ms() * 1000.0) / clockRateUs;
+        _bits.resize(bitsPerRevolution);
+        _cursor = 0;
+
+        writeFillerRawBytes(_config.post_index_gap_bytes(), 0xaaaa);
+
+        for (const auto& sector : sectors)
+        {
+            /* Header */
+
+            writeFillerRawBytes(_config.pre_sector_gap_bytes(), 0xaaaa);
+            writeRawBits(SECTOR_ID, 64);
+            writeByte(0x5a);
+            writeByte((sector->logicalTrack << 1) | sector->logicalSide);
+            writeByte(sector->logicalSector);
+            writeByte(0x5a);
+
+            /* Data */
+
+            writeFillerRawBytes(_config.pre_data_gap_bytes(), 0xaaaa);
+            auto data = sector->data.slice(0, AGAT_SECTOR_SIZE);
+            writeRawBits(DATA_ID, 64);
+            writeBytes(data);
+            writeByte(agatChecksum(data));
+            writeByte(0x5a);
+        }
+
+        if (_cursor >= _bits.size())
+            error("track data overrun");
+        fillBitmapTo(_bits, _cursor, _bits.size(), {true, false});
+
+        auto fluxmap = std::make_unique<Fluxmap>();
+        fluxmap->appendBits(_bits,
+            calculatePhysicalClockPeriod(_config.target_clock_period_us() * 1e3,
+                _config.target_rotational_period_ms() * 1e6));
+        return fluxmap;
+    }
+
+private:
+    const AgatEncoderProto& _config;
+    uint32_t _cursor;
+    bool _lastBit;
+    std::vector<bool> _bits;
+};
+
+std::unique_ptr<Encoder> createAgatEncoder(const EncoderProto& config)
+{
+    return std::unique_ptr<Encoder>(new AgatEncoder(config));
+}

arch/amiga/amiga.cc

@@ -1,7 +1,7 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "amiga.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/amiga/amiga.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 
 uint32_t amigaChecksum(const Bytes& bytes)
@@ -18,61 +18,61 @@ uint32_t amigaChecksum(const Bytes& bytes)
 
 static uint8_t everyother(uint16_t x)
 {
-	                  /* aabb ccdd eeff gghh */
-	x &= 0x6666;      /* 0ab0 0cd0 0ef0 0gh0 */
-	x >>= 1;          /* 00ab 00cd 00ef 00gh */
-	x |= x << 2;      /* abab cdcd efef ghgh */
-	x &= 0x3c3c;      /* 00ab cd00 00ef gh00 */
-	x >>= 2;          /* 0000 abcd 0000 efgh */
-	x |= x >> 4;      /* 0000 abcd abcd efgh */
-	return x;
+    /* aabb ccdd eeff gghh */
+    x &= 0x6666; /* 0ab0 0cd0 0ef0 0gh0 */
+    x >>= 1;     /* 00ab 00cd 00ef 00gh */
+    x |= x << 2; /* abab cdcd efef ghgh */
+    x &= 0x3c3c; /* 00ab cd00 00ef gh00 */
+    x >>= 2;     /* 0000 abcd 0000 efgh */
+    x |= x >> 4; /* 0000 abcd abcd efgh */
+    return x;
 }
 
 Bytes amigaInterleave(const Bytes& input)
 {
-	Bytes output;
-	ByteWriter bw(output);
+    Bytes output;
+    ByteWriter bw(output);
 
-	/* Write all odd bits. (Numbering starts at 0...) */
+    /* Write all odd bits. (Numbering starts at 0...) */
 
-	{
-		ByteReader br(input);
-		while (!br.eof())
-		{
-			uint16_t x = br.read_be16();
-			x &= 0xaaaa;       /* a0b0 c0d0 e0f0 g0h0 */
-			x |= x >> 1;       /* aabb ccdd eeff gghh */
-			x = everyother(x); /* 0000 0000 abcd efgh */
-			bw.write_8(x);
-		}
-	}
+    {
+        ByteReader br(input);
+        while (!br.eof())
+        {
+            uint16_t x = br.read_be16();
+            x &= 0xaaaa;       /* a0b0 c0d0 e0f0 g0h0 */
+            x |= x >> 1;       /* aabb ccdd eeff gghh */
+            x = everyother(x); /* 0000 0000 abcd efgh */
+            bw.write_8(x);
+        }
+    }
 
-	/* Write all even bits. */
+    /* Write all even bits. */
 
-	{
-		ByteReader br(input);
-		while (!br.eof())
-		{
-			uint16_t x = br.read_be16();
-			x &= 0x5555;       /* 0a0b 0c0d 0e0f 0g0h */
-			x |= x << 1;       /* aabb ccdd eeff gghh */
-			x = everyother(x); /* 0000 0000 abcd efgh */
-			bw.write_8(x);
-		}
-	}
+    {
+        ByteReader br(input);
+        while (!br.eof())
+        {
+            uint16_t x = br.read_be16();
+            x &= 0x5555;       /* 0a0b 0c0d 0e0f 0g0h */
+            x |= x << 1;       /* aabb ccdd eeff gghh */
+            x = everyother(x); /* 0000 0000 abcd efgh */
+            bw.write_8(x);
+        }
+    }
 
-	return output;
+    return output;
 }
 
 Bytes amigaDeinterleave(const uint8_t*& input, size_t len)
 {
     assert(!(len & 1));
     const uint8_t* odds = &input[0];
-    const uint8_t* evens = &input[len/2];
+    const uint8_t* evens = &input[len / 2];
     Bytes output;
     ByteWriter bw(output);
 
-    for (size_t i=0; i<len/2; i++)
+    for (size_t i = 0; i < len / 2; i++)
     {
         uint8_t o = *odds++;
         uint8_t e = *evens++;
@@ -81,11 +81,15 @@ Bytes amigaDeinterleave(const uint8_t*& input, size_t len)
          * http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
          */
         uint16_t result =
-            (((e * 0x0101010101010101ULL & 0x8040201008040201ULL)
-                * 0x0102040810204081ULL >> 49) & 0x5555) |
-            (((o * 0x0101010101010101ULL & 0x8040201008040201ULL)
-                * 0x0102040810204081ULL >> 48) & 0xAAAA);
-        
+            (((e * 0x0101010101010101ULL & 0x8040201008040201ULL) *
+                     0x0102040810204081ULL >>
+                 49) &
+                0x5555) |
+            (((o * 0x0101010101010101ULL & 0x8040201008040201ULL) *
+                     0x0102040810204081ULL >>
+                 48) &
+                0xAAAA);
+
         bw.write_be16(result);
     }
 
@@ -95,6 +99,6 @@ Bytes amigaDeinterleave(const uint8_t*& input, size_t len)
 
 Bytes amigaDeinterleave(const Bytes& input)
 {
-	const uint8_t* ptr = input.cbegin();
-	return amigaDeinterleave(ptr, input.size());
+    const uint8_t* ptr = input.cbegin();
+    return amigaDeinterleave(ptr, input.size());
 }

arch/amiga/amiga.h

@@ -1,7 +1,7 @@
 #ifndef AMIGA_H
 #define AMIGA_H
 
-#include "encoders/encoders.h"
+#include "lib/encoders/encoders.h"
 
 #define AMIGA_SECTOR_RECORD 0xaaaa44894489LL
 

arch/amiga/amiga.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message AmigaDecoderProto {}
 

arch/amiga/decoder.cc

@@ -1,80 +1,85 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "amiga.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/amiga/amiga.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include "lib/decoders/decoders.pb.h"
 #include <string.h>
 #include <algorithm>
 
-/* 
+/*
  * Amiga disks use MFM but it's not quite the same as IBM MFM. They only use
  * a single type of record with a different marker byte.
- * 
+ *
  * See the big comment in the IBM MFM decoder for the gruesome details of how
  * MFM works.
  */
-         
+
 static const FluxPattern SECTOR_PATTERN(48, AMIGA_SECTOR_RECORD);
 
 class AmigaDecoder : public Decoder
 {
 public:
-	AmigaDecoder(const DecoderProto& config):
-		Decoder(config),
-		_config(config.amiga())
-	{}
+    AmigaDecoder(const DecoderProto& config):
+        Decoder(config),
+        _config(config.amiga())
+    {
+    }
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(SECTOR_PATTERN);
-	}
+    {
+        return seekToPattern(SECTOR_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		if (readRaw48() != AMIGA_SECTOR_RECORD)
-			return;
-			
-		const auto& rawbits = readRawBits(AMIGA_RECORD_SIZE*16);
-		if (rawbits.size() < (AMIGA_RECORD_SIZE*16))
-			return;
-		const auto& rawbytes = toBytes(rawbits).slice(0, AMIGA_RECORD_SIZE*2);
-		const auto& bytes = decodeFmMfm(rawbits).slice(0, AMIGA_RECORD_SIZE);
+    {
+        if (readRaw48() != AMIGA_SECTOR_RECORD)
+            return;
 
-		const uint8_t* ptr = bytes.begin();
+        const auto& rawbits = readRawBits(AMIGA_RECORD_SIZE * 16);
+        if (rawbits.size() < (AMIGA_RECORD_SIZE * 16))
+            return;
+        const auto& rawbytes = toBytes(rawbits).slice(0, AMIGA_RECORD_SIZE * 2);
+        const auto& bytes = decodeFmMfm(rawbits).slice(0, AMIGA_RECORD_SIZE);
 
-		Bytes header = amigaDeinterleave(ptr, 4);
-		Bytes recoveryinfo = amigaDeinterleave(ptr, 16);
+        const uint8_t* ptr = bytes.begin();
 
-		_sector->logicalTrack = header[1] >> 1;
-		_sector->logicalSide = header[1] & 1;
-		_sector->logicalSector = header[2];
+        Bytes header = amigaDeinterleave(ptr, 4);
+        Bytes recoveryinfo = amigaDeinterleave(ptr, 16);
 
-		uint32_t wantedheaderchecksum = amigaDeinterleave(ptr, 4).reader().read_be32();
-		uint32_t gotheaderchecksum = amigaChecksum(rawbytes.slice(0, 40));
-		if (gotheaderchecksum != wantedheaderchecksum)
-			return;
+        _sector->logicalTrack = header[1] >> 1;
+        _sector->logicalSide = header[1] & 1;
+        _sector->logicalSector = header[2];
 
-		uint32_t wanteddatachecksum = amigaDeinterleave(ptr, 4).reader().read_be32();
-		uint32_t gotdatachecksum = amigaChecksum(rawbytes.slice(56, 1024));
+        uint32_t wantedheaderchecksum =
+            amigaDeinterleave(ptr, 4).reader().read_be32();
+        uint32_t gotheaderchecksum = amigaChecksum(rawbytes.slice(0, 40));
+        if (gotheaderchecksum != wantedheaderchecksum)
+            return;
 
-		Bytes data;
-		data.writer().append(amigaDeinterleave(ptr, 512)).append(recoveryinfo);
-		_sector->data = data;
-		_sector->status = (gotdatachecksum == wanteddatachecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        uint32_t wanteddatachecksum =
+            amigaDeinterleave(ptr, 4).reader().read_be32();
+        uint32_t gotdatachecksum = amigaChecksum(rawbytes.slice(56, 1024));
+
+        Bytes data;
+        data.writer().append(amigaDeinterleave(ptr, 512)).append(recoveryinfo);
+        _sector->data = data;
+        _sector->status = (gotdatachecksum == wanteddatachecksum)
+                              ? Sector::OK
+                              : Sector::BAD_CHECKSUM;
+    }
 
 private:
-	const AmigaDecoderProto& _config;
-	nanoseconds_t _clock;
+    const AmigaDecoderProto& _config;
+    nanoseconds_t _clock;
 };
 
 std::unique_ptr<Decoder> createAmigaDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new AmigaDecoder(config));
+    return std::unique_ptr<Decoder>(new AmigaDecoder(config));
 }
-

arch/amiga/encoder.cc

@@ -1,10 +1,10 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "amiga.h"
-#include "crc.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/amiga/amiga.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
 #include "arch/amiga/amiga.pb.h"
 #include "lib/encoders/encoders.pb.h"
 
@@ -59,7 +59,7 @@ static void write_sector(std::vector<bool>& bits,
     const std::shared_ptr<const Sector>& sector)
 {
     if ((sector->data.size() != 512) && (sector->data.size() != 528))
-        Error() << "unsupported sector size --- you must pick 512 or 528";
+        error("unsupported sector size --- you must pick 512 or 528");
 
     uint32_t checksum = 0;
 
@@ -114,7 +114,8 @@ public:
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
-        /* Number of bits for one nominal revolution of a real 200ms Amiga disk. */
+        /* Number of bits for one nominal revolution of a real 200ms Amiga disk.
+         */
         int bitsPerRevolution = 200e3 / _config.clock_rate_us();
         std::vector<bool> bits(bitsPerRevolution);
         unsigned cursor = 0;
@@ -129,13 +130,12 @@ public:
             write_sector(bits, cursor, sector);
 
         if (cursor >= bits.size())
-            Error() << "track data overrun";
+            error("track data overrun");
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         auto fluxmap = std::make_unique<Fluxmap>();
         fluxmap->appendBits(bits,
-            calculatePhysicalClockPeriod(
-                _config.clock_rate_us() * 1e3, 200e6));
+            calculatePhysicalClockPeriod(_config.clock_rate_us() * 1e3, 200e6));
         return fluxmap;
     }
 

arch/apple2/apple2.h

@@ -2,19 +2,18 @@
 #define APPLE2_H
 
 #include <memory.h>
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
 
-#define APPLE2_SECTOR_RECORD   0xd5aa96
-#define APPLE2_DATA_RECORD     0xd5aaad
+#define APPLE2_SECTOR_RECORD 0xd5aa96
+#define APPLE2_DATA_RECORD 0xd5aaad
 
-#define APPLE2_SECTOR_LENGTH   256
+#define APPLE2_SECTOR_LENGTH 256
 #define APPLE2_ENCODED_SECTOR_LENGTH 342
 
-#define APPLE2_SECTORS         16
+#define APPLE2_SECTORS 16
 
 extern std::unique_ptr<Decoder> createApple2Decoder(const DecoderProto& config);
 extern std::unique_ptr<Encoder> createApple2Encoder(const EncoderProto& config);
 
 #endif
-

arch/apple2/apple2.proto

@@ -1,8 +1,11 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
-message Apple2DecoderProto {}
+message Apple2DecoderProto {
+	optional uint32 side_one_track_offset = 1
+		[ default = 0, (help) = "offset to apply to track numbers on side 1" ];
+}
 
 message Apple2EncoderProto
 {
@@ -13,4 +16,7 @@ message Apple2EncoderProto
     /* Apple II disk drives spin at 300rpm. */
     optional double rotational_period_ms = 2
         [ default = 200.0, (help) = "rotational period on the real device" ];
+
+	optional uint32 side_one_track_offset = 3
+		[ default = 0, (help) = "offset to apply to track numbers on side 1" ];
 }

arch/apple2/decoder.cc

@@ -1,33 +1,39 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "apple2.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/apple2/apple2.h"
+#include "arch/apple2/apple2.pb.h"
+#include "lib/decoders/decoders.pb.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
 const FluxPattern SECTOR_RECORD_PATTERN(24, APPLE2_SECTOR_RECORD);
 const FluxPattern DATA_RECORD_PATTERN(24, APPLE2_DATA_RECORD);
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
     {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "data_gcr.h"
+#undef GCR_ENTRY
     }
     return -1;
 }
 
-/* This is extremely inspired by the MESS implementation, written by Nathan Woods
- * and R. Belmont: https://github.com/mamedev/mame/blob/7914a6083a3b3a8c243ae6c3b8cb50b023f21e0e/src/lib/formats/ap2_dsk.cpp
+/* This is extremely inspired by the MESS implementation, written by Nathan
+ * Woods and R. Belmont:
+ * https://github.com/mamedev/mame/blob/7914a6083a3b3a8c243ae6c3b8cb50b023f21e0e/src/lib/formats/ap2_dsk.cpp
  */
 static Bytes decode_crazy_data(const uint8_t* inp, Sector::Status& status)
 {
@@ -47,9 +53,11 @@ static Bytes decode_crazy_data(const uint8_t* inp, Sector::Status& status)
         {
             /* 3 * 2 bit */
             output[i + 0] = ((checksum >> 1) & 0x01) | ((checksum << 1) & 0x02);
-            output[i + 86] = ((checksum >> 3) & 0x01) | ((checksum >> 1) & 0x02);
+            output[i + 86] =
+                ((checksum >> 3) & 0x01) | ((checksum >> 1) & 0x02);
             if ((i + 172) < APPLE2_SECTOR_LENGTH)
-                output[i + 172] = ((checksum >> 5) & 0x01) | ((checksum >> 3) & 0x02);
+                output[i + 172] =
+                    ((checksum >> 5) & 0x01) | ((checksum >> 3) & 0x02);
         }
     }
 
@@ -67,88 +75,102 @@ static uint8_t combine(uint16_t word)
 class Apple2Decoder : public Decoder
 {
 public:
-	Apple2Decoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    Apple2Decoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		if (readRaw24() != APPLE2_SECTOR_RECORD)
-			return;
+    {
+        if (readRaw24() != APPLE2_SECTOR_RECORD)
+            return;
 
-		/* Read header. */
+        /* Read header. */
 
-		auto header = toBytes(readRawBits(8*8)).slice(0, 8);
-		ByteReader br(header);
+        auto header = toBytes(readRawBits(8 * 8)).slice(0, 8);
+        ByteReader br(header);
 
-		uint8_t volume = combine(br.read_be16());
-		_sector->logicalTrack = combine(br.read_be16());
-		_sector->logicalSector = combine(br.read_be16());
-		uint8_t checksum = combine(br.read_be16());
+        uint8_t volume = combine(br.read_be16());
+        _sector->logicalTrack = combine(br.read_be16());
+        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalSector = combine(br.read_be16());
+        uint8_t checksum = combine(br.read_be16());
 
-		// If the checksum is correct, upgrade the sector from MISSING
-		// to DATA_MISSING in anticipation of its data record
-		if (checksum == (volume ^ _sector->logicalTrack ^ _sector->logicalSector))
-			_sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
-	}
+        // If the checksum is correct, upgrade the sector from MISSING
+        // to DATA_MISSING in anticipation of its data record
+        if (checksum ==
+            (volume ^ _sector->logicalTrack ^ _sector->logicalSector))
+            _sector->status =
+                Sector::DATA_MISSING; /* unintuitive but correct */
+
+        if (_sector->logicalSide == 1)
+            _sector->logicalTrack -= _config.apple2().side_one_track_offset();
+
+        /* Sanity check. */
+
+        if (_sector->logicalTrack > 100)
+        {
+            _sector->status = Sector::MISSING;
+            return;
+        }
+    }
 
     void decodeDataRecord() override
-	{
-		/* Check ID. */
+    {
+        /* Check ID. */
 
-		if (readRaw24() != APPLE2_DATA_RECORD)
-			return;
+        if (readRaw24() != APPLE2_DATA_RECORD)
+            return;
 
-		// Sometimes there's a 1-bit gap between APPLE2_DATA_RECORD and
-		// the data itself.  This has been seen on real world disks
-		// such as the Apple II Operating System Kit from Apple2Online.
-		// However, I haven't seen it described in any of the various
-		// references.
-		//
-		// This extra '0' bit would not affect the real disk interface,
-		// as it was a '1' reaching the top bit of a shift register
-		// that triggered a byte to be available, but it affects the
-		// way the data is read here.
-		//
-		// While the floppies tested only seemed to need this applied
-		// to the first byte of the data record, applying it
-		// consistently to all of them doesn't seem to hurt, and
-		// simplifies the code.
+        // Sometimes there's a 1-bit gap between APPLE2_DATA_RECORD and
+        // the data itself.  This has been seen on real world disks
+        // such as the Apple II Operating System Kit from Apple2Online.
+        // However, I haven't seen it described in any of the various
+        // references.
+        //
+        // This extra '0' bit would not affect the real disk interface,
+        // as it was a '1' reaching the top bit of a shift register
+        // that triggered a byte to be available, but it affects the
+        // way the data is read here.
+        //
+        // While the floppies tested only seemed to need this applied
+        // to the first byte of the data record, applying it
+        // consistently to all of them doesn't seem to hurt, and
+        // simplifies the code.
 
-		/* Read and decode data. */
+        /* Read and decode data. */
 
-		auto readApple8 = [&]() {
-		    auto result = 0;
-		    while((result & 0x80) == 0) {
-			auto b = readRawBits(1);
-                        if(b.empty()) break;
-			result = (result << 1) | b[0];
-		    }
-		    return result;
-		};
+        auto readApple8 = [&]()
+        {
+            auto result = 0;
+            while ((result & 0x80) == 0)
+            {
+                auto b = readRawBits(1);
+                if (b.empty())
+                    break;
+                result = (result << 1) | b[0];
+            }
+            return result;
+        };
 
-		constexpr unsigned recordLength = APPLE2_ENCODED_SECTOR_LENGTH+2;
-                uint8_t bytes[recordLength];
-                for(auto &byte : bytes) {
-                    byte = readApple8();
-                }
+        constexpr unsigned recordLength = APPLE2_ENCODED_SECTOR_LENGTH + 2;
+        uint8_t bytes[recordLength];
+        for (auto& byte : bytes)
+        {
+            byte = readApple8();
+        }
 
-		// Upgrade the sector from MISSING to BAD_CHECKSUM.
-		// If decode_crazy_data succeeds, it upgrades the sector to
-		// OK.
-		_sector->status = Sector::BAD_CHECKSUM;
-		_sector->data = decode_crazy_data(&bytes[0], _sector->status);
-	}
+        // Upgrade the sector from MISSING to BAD_CHECKSUM.
+        // If decode_crazy_data succeeds, it upgrades the sector to
+        // OK.
+        _sector->status = Sector::BAD_CHECKSUM;
+        _sector->data = decode_crazy_data(&bytes[0], _sector->status);
+    }
 };
 
 std::unique_ptr<Decoder> createApple2Decoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new Apple2Decoder(config));
+    return std::unique_ptr<Decoder>(new Apple2Decoder(config));
 }
-
-

arch/apple2/encoder.cc

@@ -1,14 +1,14 @@
-#include "globals.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
 #include "arch/apple2/apple2.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "sector.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "lib/data/sector.h"
+#include "lib/data/image.h"
 #include "fmt/format.h"
 #include "lib/encoders/encoders.pb.h"
 #include <ctype.h>
-#include "bytes.h"
+#include "lib/core/bytes.h"
 
 static int encode_data_gcr(uint8_t data)
 {
@@ -50,14 +50,12 @@ public:
             writeSector(bits, cursor, *sector);
 
         if (cursor >= bits.size())
-            Error() << fmt::format(
-                "track data overrun by {} bits", cursor - bits.size());
+            error("track data overrun by {} bits", cursor - bits.size());
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
         fluxmap->appendBits(bits,
-            calculatePhysicalClockPeriod(
-                _config.clock_period_us() * 1e3,
+            calculatePhysicalClockPeriod(_config.clock_period_us() * 1e3,
                 _config.rotational_period_ms() * 1e6));
         return fluxmap;
     }
@@ -119,8 +117,7 @@ private:
 
             // There is data to encode to disk.
             if ((sector.data.size() != APPLE2_SECTOR_LENGTH))
-                Error() << fmt::format(
-                    "unsupported sector size {} --- you must pick 256",
+                error("unsupported sector size {} --- you must pick 256",
                     sector.data.size());
 
             // Write address syncing leader : A sequence of "FF40"s; 5 of them
@@ -132,13 +129,17 @@ private:
             // extra padding.
             write_ff40(sector.logicalSector == 0 ? 32 : 8);
 
+            int track = sector.logicalTrack;
+            if (sector.logicalSide == 1)
+                track += _config.side_one_track_offset();
+
             // Write address field: APPLE2_SECTOR_RECORD + sector identifier +
             // DE AA EB
             write_bits(APPLE2_SECTOR_RECORD, 24);
             write_gcr44(volume_id);
-            write_gcr44(sector.logicalTrack);
+            write_gcr44(track);
             write_gcr44(sector.logicalSector);
-            write_gcr44(volume_id ^ sector.logicalTrack ^ sector.logicalSector);
+            write_gcr44(volume_id ^ track ^ sector.logicalSector);
             write_bits(0xDEAAEB, 24);
 
             // Write data syncing leader: FF40 + APPLE2_DATA_RECORD + sector

arch/arch.cc

@@ -0,0 +1,97 @@
+#include "lib/core/globals.h"
+#include "lib/encoders/encoders.h"
+#include "lib/decoders/decoders.h"
+#include "lib/config/config.h"
+#include "arch/agat/agat.h"
+#include "arch/aeslanier/aeslanier.h"
+#include "arch/amiga/amiga.h"
+#include "arch/apple2/apple2.h"
+#include "arch/brother/brother.h"
+#include "arch/c64/c64.h"
+#include "arch/f85/f85.h"
+#include "arch/fb100/fb100.h"
+#include "arch/ibm/ibm.h"
+#include "arch/macintosh/macintosh.h"
+#include "arch/micropolis/micropolis.h"
+#include "arch/mx/mx.h"
+#include "arch/northstar/northstar.h"
+#include "arch/rolandd20/rolandd20.h"
+#include "arch/smaky6/smaky6.h"
+#include "arch/tartu/tartu.h"
+#include "arch/tids990/tids990.h"
+#include "arch/victor9k/victor9k.h"
+#include "arch/zilogmcz/zilogmcz.h"
+#include "arch/arch.h"
+
+std::unique_ptr<Encoder> Arch::createEncoder(Config& config)
+{
+    if (!config.hasEncoder())
+        error("no encoder configured");
+    return createEncoder(config->encoder());
+}
+
+std::unique_ptr<Encoder> Arch::createEncoder(const EncoderProto& config)
+{
+    static const std::map<int,
+        std::function<std::unique_ptr<Encoder>(const EncoderProto&)>>
+        encoders = {
+            {EncoderProto::kAgat,       createAgatEncoder       },
+            {EncoderProto::kAmiga,      createAmigaEncoder      },
+            {EncoderProto::kApple2,     createApple2Encoder     },
+            {EncoderProto::kBrother,    createBrotherEncoder    },
+            {EncoderProto::kC64,        createCommodore64Encoder},
+            {EncoderProto::kIbm,        createIbmEncoder        },
+            {EncoderProto::kMacintosh,  createMacintoshEncoder  },
+            {EncoderProto::kMicropolis, createMicropolisEncoder },
+            {EncoderProto::kNorthstar,  createNorthstarEncoder  },
+            {EncoderProto::kTartu,      createTartuEncoder      },
+            {EncoderProto::kTids990,    createTids990Encoder    },
+            {EncoderProto::kVictor9K,   createVictor9kEncoder   },
+    };
+
+    auto encoder = encoders.find(config.format_case());
+    if (encoder == encoders.end())
+        error("no encoder specified");
+
+    return (encoder->second)(config);
+}
+
+std::unique_ptr<Decoder> Arch::createDecoder(Config& config)
+{
+    if (!config.hasDecoder())
+        error("no decoder configured");
+    return createDecoder(config->decoder());
+}
+
+std::unique_ptr<Decoder> Arch::createDecoder(const DecoderProto& config)
+{
+    static const std::map<int,
+        std::function<std::unique_ptr<Decoder>(const DecoderProto&)>>
+        decoders = {
+            {DecoderProto::kAgat,       createAgatDecoder       },
+            {DecoderProto::kAeslanier,  createAesLanierDecoder  },
+            {DecoderProto::kAmiga,      createAmigaDecoder      },
+            {DecoderProto::kApple2,     createApple2Decoder     },
+            {DecoderProto::kBrother,    createBrotherDecoder    },
+            {DecoderProto::kC64,        createCommodore64Decoder},
+            {DecoderProto::kF85,        createDurangoF85Decoder },
+            {DecoderProto::kFb100,      createFb100Decoder      },
+            {DecoderProto::kIbm,        createIbmDecoder        },
+            {DecoderProto::kMacintosh,  createMacintoshDecoder  },
+            {DecoderProto::kMicropolis, createMicropolisDecoder },
+            {DecoderProto::kMx,         createMxDecoder         },
+            {DecoderProto::kNorthstar,  createNorthstarDecoder  },
+            {DecoderProto::kRolandd20,  createRolandD20Decoder  },
+            {DecoderProto::kSmaky6,     createSmaky6Decoder     },
+            {DecoderProto::kTartu,      createTartuDecoder      },
+            {DecoderProto::kTids990,    createTids990Decoder    },
+            {DecoderProto::kVictor9K,   createVictor9kDecoder   },
+            {DecoderProto::kZilogmcz,   createZilogMczDecoder   },
+    };
+
+    auto decoder = decoders.find(config.format_case());
+    if (decoder == decoders.end())
+        error("no decoder specified");
+
+    return (decoder->second)(config);
+}

arch/arch.h

@@ -0,0 +1,16 @@
+#pragma once
+
+class Encoder;
+class Decoder;
+class DecoderProto;
+class EncoderProto;
+class Config;
+
+namespace Arch
+{
+    std::unique_ptr<Decoder> createDecoder(Config& config);
+    std::unique_ptr<Decoder> createDecoder(const DecoderProto& config);
+
+    std::unique_ptr<Encoder> createEncoder(Config& config);
+    std::unique_ptr<Encoder> createEncoder(const EncoderProto& config);
+}

arch/brother/brother.h

@@ -3,17 +3,19 @@
 
 /* Brother word processor format (or at least, one of them) */
 
-#define BROTHER_SECTOR_RECORD            0xFFFFFD57
-#define BROTHER_DATA_RECORD              0xFFFFFDDB
-#define BROTHER_DATA_RECORD_PAYLOAD      256
-#define BROTHER_DATA_RECORD_CHECKSUM     3
+#define BROTHER_SECTOR_RECORD 0xFFFFFD57
+#define BROTHER_DATA_RECORD 0xFFFFFDDB
+#define BROTHER_DATA_RECORD_PAYLOAD 256
+#define BROTHER_DATA_RECORD_CHECKSUM 3
 #define BROTHER_DATA_RECORD_ENCODED_SIZE 415
 
-#define BROTHER_TRACKS_PER_240KB_DISK    78
-#define BROTHER_TRACKS_PER_120KB_DISK    39
-#define BROTHER_SECTORS_PER_TRACK        12
+#define BROTHER_TRACKS_PER_240KB_DISK 78
+#define BROTHER_TRACKS_PER_120KB_DISK 39
+#define BROTHER_SECTORS_PER_TRACK 12
 
-extern std::unique_ptr<Decoder> createBrotherDecoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createBrotherEncoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createBrotherDecoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createBrotherEncoder(
+    const EncoderProto& config);
 
 #endif

arch/brother/data_gcr.h

@@ -1,13 +1,13 @@
-GCR_ENTRY(0x55, 0) // 00000
-GCR_ENTRY(0x57, 1) // 00001
-GCR_ENTRY(0x5b, 2) // 00010
-GCR_ENTRY(0x5d, 3) // 00011
-GCR_ENTRY(0x5f, 4) // 00100 
-GCR_ENTRY(0x6b, 5) // 00101
-GCR_ENTRY(0x6d, 6) // 00110
-GCR_ENTRY(0x6f, 7) // 00111
-GCR_ENTRY(0x75, 8) // 01000
-GCR_ENTRY(0x77, 9) // 01001
+GCR_ENTRY(0x55, 0)  // 00000
+GCR_ENTRY(0x57, 1)  // 00001
+GCR_ENTRY(0x5b, 2)  // 00010
+GCR_ENTRY(0x5d, 3)  // 00011
+GCR_ENTRY(0x5f, 4)  // 00100
+GCR_ENTRY(0x6b, 5)  // 00101
+GCR_ENTRY(0x6d, 6)  // 00110
+GCR_ENTRY(0x6f, 7)  // 00111
+GCR_ENTRY(0x75, 8)  // 01000
+GCR_ENTRY(0x77, 9)  // 01001
 GCR_ENTRY(0x7b, 10) // 01010
 GCR_ENTRY(0x7d, 11) // 01011
 GCR_ENTRY(0x7f, 12) // 01100
@@ -30,4 +30,3 @@ GCR_ENTRY(0xef, 28) // 11100
 GCR_ENTRY(0xf5, 29) // 11101
 GCR_ENTRY(0xf7, 30) // 11110
 GCR_ENTRY(0xfb, 31) // 11111
-

arch/brother/decoder.cc

@@ -1,17 +1,19 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "brother.h"
-#include "sector.h"
-#include "bytes.h"
-#include "crc.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/brother/brother.h"
+#include "lib/data/sector.h"
+#include "lib/core/bytes.h"
+#include "lib/core/crc.h"
 #include <ctype.h>
 
 const FluxPattern SECTOR_RECORD_PATTERN(32, BROTHER_SECTOR_RECORD);
 const FluxPattern DATA_RECORD_PATTERN(32, BROTHER_DATA_RECORD);
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 static std::vector<uint8_t> outputbuffer;
 
@@ -32,88 +34,89 @@ static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
     {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "data_gcr.h"
+#undef GCR_ENTRY
     }
     return -1;
 }
 
 static int decode_header_gcr(uint16_t word)
 {
-	switch (word)
-	{
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "header_gcr.h"
-		#undef GCR_ENTRY
-	}                       
-	return -1;             
+    switch (word)
+    {
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "header_gcr.h"
+#undef GCR_ENTRY
+    }
+    return -1;
 }
 
 class BrotherDecoder : public Decoder
 {
 public:
-    BrotherDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    BrotherDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		if (readRaw32() != BROTHER_SECTOR_RECORD)
-			return;
+    {
+        if (readRaw32() != BROTHER_SECTOR_RECORD)
+            return;
 
-		const auto& rawbits = readRawBits(32);
-		const auto& bytes = toBytes(rawbits).slice(0, 4);
+        const auto& rawbits = readRawBits(32);
+        const auto& bytes = toBytes(rawbits).slice(0, 4);
 
-		ByteReader br(bytes);
-		_sector->logicalTrack = decode_header_gcr(br.read_be16());
-		_sector->logicalSector = decode_header_gcr(br.read_be16());
+        ByteReader br(bytes);
+        _sector->logicalTrack = decode_header_gcr(br.read_be16());
+        _sector->logicalSector = decode_header_gcr(br.read_be16());
 
-		/* Sanity check the values read; there's no header checksum and
-			* occasionally we get garbage due to bit errors. */
-		if (_sector->logicalSector > 11)
-			return;
-		if (_sector->logicalTrack > 79)
-			return;
+        /* Sanity check the values read; there's no header checksum and
+         * occasionally we get garbage due to bit errors. */
+        if (_sector->logicalSector > 11)
+            return;
+        if (_sector->logicalTrack > 79)
+            return;
+
+        _sector->status = Sector::DATA_MISSING;
+    }
 
-		_sector->status = Sector::DATA_MISSING;
-	}
-	
     void decodeDataRecord() override
-	{
-		if (readRaw32() != BROTHER_DATA_RECORD)
-			return;
+    {
+        if (readRaw32() != BROTHER_DATA_RECORD)
+            return;
 
-		const auto& rawbits = readRawBits(BROTHER_DATA_RECORD_ENCODED_SIZE*8);
-		const auto& rawbytes = toBytes(rawbits).slice(0, BROTHER_DATA_RECORD_ENCODED_SIZE);
+        const auto& rawbits = readRawBits(BROTHER_DATA_RECORD_ENCODED_SIZE * 8);
+        const auto& rawbytes =
+            toBytes(rawbits).slice(0, BROTHER_DATA_RECORD_ENCODED_SIZE);
 
-		Bytes bytes;
-		ByteWriter bw(bytes);
-		BitWriter bitw(bw);
-		for (uint8_t b : rawbytes)
-		{
-			uint32_t nibble = decode_data_gcr(b);
-			bitw.push(nibble, 5);
-		}
-		bitw.flush();
+        Bytes bytes;
+        ByteWriter bw(bytes);
+        BitWriter bitw(bw);
+        for (uint8_t b : rawbytes)
+        {
+            uint32_t nibble = decode_data_gcr(b);
+            bitw.push(nibble, 5);
+        }
+        bitw.flush();
 
-		_sector->data = bytes.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
-		uint32_t realCrc = crcbrother(_sector->data);
-		uint32_t wantCrc = bytes.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
-		_sector->status = (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = bytes.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
+        uint32_t realCrc = crcbrother(_sector->data);
+        uint32_t wantCrc =
+            bytes.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
+        _sector->status =
+            (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createBrotherDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new BrotherDecoder(config));
+    return std::unique_ptr<Decoder>(new BrotherDecoder(config));
 }
-
-

arch/brother/encoder.cc

@@ -1,10 +1,10 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "brother.h"
-#include "crc.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/brother/brother.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
 #include "arch/brother/brother.pb.h"
 #include "lib/encoders/encoders.pb.h"
 
@@ -67,7 +67,7 @@ static void write_sector_data(
     int width = 0;
 
     if (data.size() != BROTHER_DATA_RECORD_PAYLOAD)
-        Error() << "unsupported sector size";
+        error("unsupported sector size");
 
     auto write_byte = [&](uint8_t byte)
     {
@@ -107,8 +107,7 @@ public:
     }
 
 public:
-    std::unique_ptr<Fluxmap> encode(
-		std::shared_ptr<const TrackInfo>& trackInfo,
+    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
         const std::vector<std::shared_ptr<const Sector>>& sectors,
         const Image& image) override
     {
@@ -116,8 +115,8 @@ public:
         std::vector<bool> bits(bitsPerRevolution);
         unsigned cursor = 0;
 
-		int sectorCount = 0;
-		for (const auto& sectorData : sectors)
+        int sectorCount = 0;
+        for (const auto& sectorData : sectors)
         {
             double headerMs = _config.post_index_gap_ms() +
                               sectorCount * _config.sector_spacing_ms();
@@ -126,16 +125,18 @@ public:
             unsigned dataCursor = dataMs * 1e3 / _config.clock_rate_us();
 
             fillBitmapTo(bits, cursor, headerCursor, {true, false});
-            write_sector_header(
-                bits, cursor, sectorData->logicalTrack, sectorData->logicalSector);
+            write_sector_header(bits,
+                cursor,
+                sectorData->logicalTrack,
+                sectorData->logicalSector);
             fillBitmapTo(bits, cursor, dataCursor, {true, false});
             write_sector_data(bits, cursor, sectorData->data);
 
-			sectorCount++;
+            sectorCount++;
         }
 
         if (cursor >= bits.size())
-            Error() << "track data overrun";
+            error("track data overrun");
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
@@ -147,8 +148,7 @@ private:
     const BrotherEncoderProto& _config;
 };
 
-std::unique_ptr<Encoder> createBrotherEncoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createBrotherEncoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new BrotherEncoder(config));
 }

arch/brother/header_gcr.h

@@ -76,4 +76,3 @@ GCR_ENTRY(0x6BAB, 74)
 GCR_ENTRY(0xAD5F, 75)
 GCR_ENTRY(0xDBED, 76)
 GCR_ENTRY(0x55BB, 77)
-

arch/build.mk

@@ -1,41 +0,0 @@
-LIBARCH_SRCS = \
-	arch/aeslanier/decoder.cc \
-	arch/amiga/amiga.cc \
-	arch/amiga/decoder.cc \
-	arch/amiga/encoder.cc \
-	arch/apple2/decoder.cc \
-	arch/apple2/encoder.cc \
-	arch/brother/decoder.cc \
-	arch/brother/encoder.cc \
-	arch/c64/c64.cc \
-	arch/c64/decoder.cc \
-	arch/c64/encoder.cc \
-	arch/f85/decoder.cc \
-	arch/fb100/decoder.cc \
-	arch/ibm/decoder.cc \
-	arch/ibm/encoder.cc \
-	arch/macintosh/decoder.cc \
-	arch/macintosh/encoder.cc \
-	arch/mx/decoder.cc \
-	arch/victor9k/decoder.cc \
-	arch/victor9k/encoder.cc \
-	arch/zilogmcz/decoder.cc \
-	arch/tids990/decoder.cc \
-	arch/tids990/encoder.cc \
-	arch/micropolis/decoder.cc \
-	arch/micropolis/encoder.cc \
-	arch/northstar/decoder.cc \
-	arch/northstar/encoder.cc \
-	arch/agat/agat.cc \
-	arch/agat/decoder.cc \
-
-LIBARCH_OBJS = $(patsubst %.cc, $(OBJDIR)/%.o, $(LIBARCH_SRCS))
-OBJS += $(LIBARCH_OBJS)
-$(LIBARCH_SRCS): | $(PROTO_HDRS)
-$(LIBARCH_SRCS): CFLAGS += $(PROTO_CFLAGS)
-LIBARCH_LIB = $(OBJDIR)/libarch.a
-$(LIBARCH_LIB): $(LIBARCH_OBJS)
-
-LIBARCH_LDFLAGS = $(LIBARCH_LIB)
-
-$(call use-pkgconfig, $(LIBARCH_LIB), $(LIBARCH_OBJS), fmt)

arch/build.py

@@ -0,0 +1,61 @@
+from build.c import cxxlibrary
+from build.protobuf import proto, protocc, protolib
+from os.path import *
+from glob import glob
+import sys
+
+archs = [f for f in glob("*", root_dir="arch") if isfile(f"arch/{f}/{f}.proto")]
+
+ps = []
+pls = []
+cls = []
+for a in archs:
+    ps += [
+        proto(
+            name=f"proto_{a}",
+            srcs=[f"arch/{a}/{a}.proto"],
+            deps=["lib/config+common_proto"],
+        )
+    ]
+
+    pls += [
+        protocc(
+            name=f"proto_lib_{a}",
+            srcs=[f".+proto_{a}"],
+            deps=["lib/config+common_proto_lib"],
+        )
+    ]
+
+    cls += [
+        cxxlibrary(
+            name=f"arch_{a}",
+            srcs=glob(f"arch/{a}/*.cc") + glob(f"arch/{a}/*.h"),
+            hdrs={f"arch/{a}/{a}.h": f"arch/{a}/{a}.h"},
+            deps=[
+                "lib/core",
+                "lib/data",
+                "lib/config",
+                "lib/encoders",
+                "lib/decoders",
+            ],
+        )
+    ]
+
+protolib(
+    name="proto",
+    srcs=ps + ["lib/config+common_proto"],
+)
+
+cxxlibrary(name="proto_lib", deps=pls)
+
+cxxlibrary(
+    name="arch",
+    srcs=[
+        "./arch.cc",
+    ],
+    hdrs={
+        "arch/arch.h": "./arch.h",
+    },
+    deps=cls
+    + ["lib/core", "lib/data", "lib/config", "lib/encoders", "lib/decoders"],
+)

arch/c64/c64.cc

@@ -1,28 +1,28 @@
-#include "globals.h"
-#include "c64.h"
+#include "lib/core/globals.h"
+#include "arch/c64/c64.h"
 
 /*
-    *   Track   Sectors/track   # Sectors   Storage in Bytes   Clock rate
-    *   -----   -------------   ---------   ----------------   ----------
-    *    1-17        21            357           7820             3.25
-    *   18-24        19            133           7170             3.5
-    *   25-30        18            108           6300             3.75
-    *   31-40(*)     17             85           6020             4
-    *                              ---
-    *                              683 (for a 35 track image)
-    * 
-    * The clock rate is normalised for a 200ms drive.
-    */
+ *   Track   Sectors/track   # Sectors   Storage in Bytes   Clock rate
+ *   -----   -------------   ---------   ----------------   ----------
+ *    1-17        21            357           7820             3.25
+ *   18-24        19            133           7170             3.5
+ *   25-30        18            108           6300             3.75
+ *   31-40(*)     17             85           6020             4
+ *                              ---
+ *                              683 (for a 35 track image)
+ *
+ * The clock rate is normalised for a 200ms drive.
+ */
 
 nanoseconds_t clockPeriodForC64Track(unsigned track)
 {
-    constexpr double BYTE_SIZE = 8.0;
+    constexpr double b = 8.0;
 
     if (track < 17)
-        return 26.0 / BYTE_SIZE;
+        return 26.0 / b;
     if (track < 24)
-        return 28.0 / BYTE_SIZE;
+        return 28.0 / b;
     if (track < 30)
-        return 30.0 / BYTE_SIZE;
-    return 32.0 / BYTE_SIZE;
+        return 30.0 / b;
+    return 32.0 / b;
 }

arch/c64/c64.h

@@ -1,14 +1,14 @@
 #ifndef C64_H
 #define C64_H
 
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
 
-#define C64_SECTOR_RECORD    0xffd49
-#define C64_DATA_RECORD      0xffd57
-#define C64_SECTOR_LENGTH    256
+#define C64_SECTOR_RECORD 0xffd49
+#define C64_DATA_RECORD 0xffd57
+#define C64_SECTOR_LENGTH 256
 
-/* Source: http://www.unusedino.de/ec64/technical/formats/g64.html 
+/* Source: http://www.unusedino.de/ec64/technical/formats/g64.html
    1. Header sync       FF FF FF FF FF (40 'on' bits, not GCR)
    2. Header info       52 54 B5 29 4B 7A 5E 95 55 55 (10 GCR bytes)
    3. Header gap        55 55 55 55 55 55 55 55 55 (9 bytes, never read)
@@ -17,18 +17,20 @@
    6. Inter-sector gap  55 55 55 55...55 55 (4 to 12 bytes, never read)
    1. Header sync       (SYNC for the next sector)
 */
-#define C64_HEADER_DATA_SYNC        0xFF
-#define C64_HEADER_BLOCK_ID         0x08
-#define C64_DATA_BLOCK_ID           0x07
-#define C64_HEADER_GAP              0x55
-#define C64_INTER_SECTOR_GAP        0x55
-#define C64_PADDING                 0x0F
+#define C64_HEADER_DATA_SYNC 0xFF
+#define C64_HEADER_BLOCK_ID 0x08
+#define C64_DATA_BLOCK_ID 0x07
+#define C64_HEADER_GAP 0x55
+#define C64_INTER_SECTOR_GAP 0x55
+#define C64_PADDING 0x0F
 
-#define C64_TRACKS_PER_DISK         40
-#define C64_BAM_TRACK               17
+#define C64_TRACKS_PER_DISK 40
+#define C64_BAM_TRACK 17
 
-extern std::unique_ptr<Decoder> createCommodore64Decoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createCommodore64Encoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createCommodore64Decoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createCommodore64Encoder(
+    const EncoderProto& config);
 
 extern nanoseconds_t clockPeriodForC64Track(unsigned track);
 

arch/c64/c64.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message Commodore64DecoderProto {}
 

arch/c64/decoder.cc

@@ -1,12 +1,13 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "c64.h"
-#include "crc.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/c64/c64.h"
+#include "lib/core/crc.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
@@ -96,8 +97,7 @@ public:
     }
 };
 
-std::unique_ptr<Decoder> createCommodore64Decoder(
-    const DecoderProto& config)
+std::unique_ptr<Decoder> createCommodore64Decoder(const DecoderProto& config)
 {
     return std::unique_ptr<Decoder>(new Commodore64Decoder(config));
 }

arch/c64/encoder.cc

@@ -1,17 +1,17 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "c64.h"
-#include "crc.h"
-#include "sector.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/c64/c64.h"
+#include "lib/core/crc.h"
+#include "lib/data/sector.h"
+#include "lib/data/image.h"
 #include "fmt/format.h"
 #include "arch/c64/c64.pb.h"
 #include "lib/encoders/encoders.pb.h"
-#include "lib/layout.h"
+#include "lib/data/layout.h"
 #include <ctype.h>
-#include "bytes.h"
+#include "lib/core/bytes.h"
 
 static bool lastBit;
 
@@ -51,26 +51,6 @@ static void write_bits(
     }
 }
 
-void bindump(std::ostream& stream, std::vector<bool>& buffer)
-{
-    size_t pos = 0;
-
-    while ((pos < buffer.size()) and (pos < 520))
-    {
-        stream << fmt::format("{:5d} : ", pos);
-        for (int i = 0; i < 40; i++)
-        {
-            if ((pos + i) < buffer.size())
-                stream << fmt::format("{:01b}", (buffer[pos + i]));
-            else
-                stream << "-- ";
-            if ((((pos + i + 1) % 8) == 0) and i != 0)
-                stream << "  ";
-        }
-        stream << std::endl;
-        pos += 40;
-    }
-}
 static std::vector<bool> encode_data(uint8_t input)
 {
     /*
@@ -214,8 +194,7 @@ public:
             writeSector(bits, cursor, sector);
 
         if (cursor >= bits.size())
-            Error() << fmt::format(
-                "track data overrun by {} bits", cursor - bits.size());
+            error("track data overrun by {} bits", cursor - bits.size());
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
@@ -243,8 +222,7 @@ private:
         {
             // There is data to encode to disk.
             if ((sector->data.size() != C64_SECTOR_LENGTH))
-                Error() << fmt::format(
-                    "unsupported sector size {} --- you must pick 256",
+                error("unsupported sector size {} --- you must pick 256",
                     sector->data.size());
 
             // 1. Write header Sync (not GCR)

arch/f85/decoder.cc

@@ -1,28 +1,31 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "f85.h"
-#include "crc.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/f85/f85.h"
+#include "lib/core/crc.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
 const FluxPattern SECTOR_RECORD_PATTERN(24, F85_SECTOR_RECORD);
 const FluxPattern DATA_RECORD_PATTERN(24, F85_DATA_RECORD);
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
     {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "data_gcr.h"
+#undef GCR_ENTRY
     }
     return -1;
 }
@@ -37,11 +40,11 @@ static Bytes decode(const std::vector<bool>& bits)
     while (ii != bits.end())
     {
         uint8_t inputfifo = 0;
-        for (size_t i=0; i<5; i++)
+        for (size_t i = 0; i < 5; i++)
         {
             if (ii == bits.end())
                 break;
-            inputfifo = (inputfifo<<1) | *ii++;
+            inputfifo = (inputfifo << 1) | *ii++;
         }
 
         bitw.push(decode_data_gcr(inputfifo), 4);
@@ -54,56 +57,55 @@ static Bytes decode(const std::vector<bool>& bits)
 class DurangoF85Decoder : public Decoder
 {
 public:
-	DurangoF85Decoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    DurangoF85Decoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		/* Skip sync bits and ID byte. */
+    {
+        /* Skip sync bits and ID byte. */
 
-		if (readRaw24() != F85_SECTOR_RECORD)
-			return;
+        if (readRaw24() != F85_SECTOR_RECORD)
+            return;
 
-		/* Read header. */
+        /* Read header. */
 
-		const auto& bytes = decode(readRawBits(6*10));
+        const auto& bytes = decode(readRawBits(6 * 10));
 
-		_sector->logicalSector = bytes[2];
-		_sector->logicalSide = 0;
-		_sector->logicalTrack = bytes[0];
+        _sector->logicalSector = bytes[2];
+        _sector->logicalSide = 0;
+        _sector->logicalTrack = bytes[0];
 
-		uint16_t wantChecksum = bytes.reader().seek(4).read_be16();
-		uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(0, 4));
-		if (wantChecksum == gotChecksum)
-			_sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
-	}
+        uint16_t wantChecksum = bytes.reader().seek(4).read_be16();
+        uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(0, 4));
+        if (wantChecksum == gotChecksum)
+            _sector->status =
+                Sector::DATA_MISSING; /* unintuitive but correct */
+    }
 
     void decodeDataRecord() override
-	{
-		/* Skip sync bits ID byte. */
+    {
+        /* Skip sync bits ID byte. */
 
-		if (readRaw24() != F85_DATA_RECORD)
-			return;
+        if (readRaw24() != F85_DATA_RECORD)
+            return;
 
-		const auto& bytes = decode(readRawBits((F85_SECTOR_LENGTH+3)*10))
-			.slice(0, F85_SECTOR_LENGTH+3);
-		ByteReader br(bytes);
+        const auto& bytes = decode(readRawBits((F85_SECTOR_LENGTH + 3) * 10))
+                                .slice(0, F85_SECTOR_LENGTH + 3);
+        ByteReader br(bytes);
 
-		_sector->data = br.read(F85_SECTOR_LENGTH);
-		uint16_t wantChecksum = br.read_be16();
-		uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, _sector->data);
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = br.read(F85_SECTOR_LENGTH);
+        uint16_t wantChecksum = br.read_be16();
+        uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, _sector->data);
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createDurangoF85Decoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new DurangoF85Decoder(config));
+    return std::unique_ptr<Decoder>(new DurangoF85Decoder(config));
 }
-

arch/f85/f85.h

@@ -2,9 +2,10 @@
 #define F85_H
 
 #define F85_SECTOR_RECORD 0xffffce /* 1111 1111 1111 1111 1100 1110 */
-#define F85_DATA_RECORD 0xffffcb /* 1111 1111 1111 1111 1100 1101 */
-#define F85_SECTOR_LENGTH    512
+#define F85_DATA_RECORD 0xffffcb   /* 1111 1111 1111 1111 1100 1101 */
+#define F85_SECTOR_LENGTH 512
 
-extern std::unique_ptr<Decoder> createDurangoF85Decoder(const DecoderProto& config);
+extern std::unique_ptr<Decoder> createDurangoF85Decoder(
+    const DecoderProto& config);
 
 #endif

arch/fb100/decoder.cc

@@ -1,23 +1,24 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "fb100.h"
-#include "crc.h"
-#include "bytes.h"
-#include "decoders/rawbits.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/fb100/fb100.h"
+#include "lib/core/crc.h"
+#include "lib/core/bytes.h"
+#include "lib/decoders/rawbits.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
 const FluxPattern SECTOR_ID_PATTERN(16, 0xabaa);
 
-/* 
+/*
  * Reverse engineered from a dump of the floppy drive's ROM. I have no idea how
  * it works.
- * 
+ *
  * LF8BA:
  *         clra
  *         staa    X00B0
@@ -100,45 +101,43 @@ static uint16_t checksum(const Bytes& bytes)
 class Fb100Decoder : public Decoder
 {
 public:
-	Fb100Decoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    Fb100Decoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(SECTOR_ID_PATTERN);
-	}
+    {
+        return seekToPattern(SECTOR_ID_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		auto rawbits = readRawBits(FB100_RECORD_SIZE*16);
+    {
+        auto rawbits = readRawBits(FB100_RECORD_SIZE * 16);
 
-		const Bytes bytes = decodeFmMfm(rawbits).slice(0, FB100_RECORD_SIZE);
-		ByteReader br(bytes);
-		br.seek(1);
-		const Bytes id = br.read(FB100_ID_SIZE);
-		uint16_t wantIdCrc = br.read_be16();
-		uint16_t gotIdCrc = checksum(id);
-		const Bytes payload = br.read(FB100_PAYLOAD_SIZE);
-		uint16_t wantPayloadCrc = br.read_be16();
-		uint16_t gotPayloadCrc = checksum(payload);
+        const Bytes bytes = decodeFmMfm(rawbits).slice(0, FB100_RECORD_SIZE);
+        ByteReader br(bytes);
+        br.seek(1);
+        const Bytes id = br.read(FB100_ID_SIZE);
+        uint16_t wantIdCrc = br.read_be16();
+        uint16_t gotIdCrc = checksum(id);
+        const Bytes payload = br.read(FB100_PAYLOAD_SIZE);
+        uint16_t wantPayloadCrc = br.read_be16();
+        uint16_t gotPayloadCrc = checksum(payload);
 
-		if (wantIdCrc != gotIdCrc)
-			return;
+        if (wantIdCrc != gotIdCrc)
+            return;
 
-		uint8_t abssector = id[2];
-		_sector->logicalTrack = abssector >> 1;
-		_sector->logicalSide = 0;
-		_sector->logicalSector = abssector & 1;
-		_sector->data.writer().append(id.slice(5, 12)).append(payload);
+        uint8_t abssector = id[2];
+        _sector->logicalTrack = abssector >> 1;
+        _sector->logicalSide = 0;
+        _sector->logicalSector = abssector & 1;
+        _sector->data.writer().append(id.slice(5, 12)).append(payload);
 
-		_sector->status = (wantPayloadCrc == gotPayloadCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->status = (wantPayloadCrc == gotPayloadCrc)
+                              ? Sector::OK
+                              : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createFb100Decoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new Fb100Decoder(config));
+    return std::unique_ptr<Decoder>(new Fb100Decoder(config));
 }
-
-

arch/fb100/fb100.h

@@ -8,4 +8,3 @@
 extern std::unique_ptr<Decoder> createFb100Decoder(const DecoderProto& config);
 
 #endif
-

arch/ibm/decoder.cc

@@ -1,13 +1,14 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "ibm.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "sector.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/ibm/ibm.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
 #include "arch/ibm/ibm.pb.h"
-#include "proto.h"
-#include "lib/layout.h"
+#include "lib/config/proto.h"
+#include "lib/data/layout.h"
 #include <string.h>
 
 static_assert(std::is_trivially_copyable<IbmIdam>::value,
@@ -147,6 +148,7 @@ public:
         _sector->logicalSide = br.read_8();
         _sector->logicalSector = br.read_8();
         _currentSectorSize = 1 << (br.read_8() + 7);
+
         uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, br.pos));
         uint16_t wantCrc = br.read_be16();
         if (wantCrc == gotCrc)
@@ -206,6 +208,18 @@ public:
         uint16_t wantCrc = br.read_be16();
         _sector->status =
             (wantCrc == gotCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+
+        auto layout = Layout::getLayoutOfTrack(
+            _sector->logicalTrack, _sector->logicalSide);
+        if (_currentSectorSize != layout->sectorSize)
+            std::cerr << fmt::format(
+                "Warning: configured sector size for t{}.h{}.s{} is {} bytes "
+                "but that seen on disk is {} bytes\n",
+                _sector->logicalTrack,
+                _sector->logicalSide,
+                _sector->logicalSector,
+                layout->sectorSize,
+                _currentSectorSize);
     }
 
 private:

arch/ibm/encoder.cc

@@ -1,15 +1,15 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "ibm.h"
-#include "crc.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/config/config.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/ibm/ibm.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
 #include "arch/ibm/ibm.pb.h"
 #include "lib/encoders/encoders.pb.h"
 #include "fmt/format.h"
-#include "lib/proto.h"
-#include "lib/layout.h"
+#include "lib/config/proto.h"
+#include "lib/data/layout.h"
 #include <ctype.h>
 
 /* IAM record separator:
@@ -112,10 +112,11 @@ public:
         const Image& image) override
     {
         IbmEncoderProto::TrackdataProto trackdata;
-        getEncoderTrackData(trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
+        getEncoderTrackData(
+            trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
 
-        auto trackLayout =
-            Layout::getLayoutOfTrack(trackInfo->logicalTrack, trackInfo->logicalSide);
+        auto trackLayout = Layout::getLayoutOfTrack(
+            trackInfo->logicalTrack, trackInfo->logicalSide);
 
         auto writeBytes = [&](const Bytes& bytes)
         {
@@ -257,7 +258,7 @@ public:
         }
 
         if (_cursor >= _bits.size())
-            Error() << "track data overrun";
+            error("track data overrun");
         while (_cursor < _bits.size())
             writeFillerRawBytes(1, gapFill);
 

arch/ibm/ibm.h

@@ -31,9 +31,7 @@ class Decoder;
 class DecoderProto;
 class EncoderProto;
 
-extern std::unique_ptr<Decoder> createIbmDecoder(
-    const DecoderProto& config);
-extern std::unique_ptr<Encoder> createIbmEncoder(
-    const EncoderProto& config);
+extern std::unique_ptr<Decoder> createIbmDecoder(const DecoderProto& config);
+extern std::unique_ptr<Encoder> createIbmEncoder(const EncoderProto& config);
 
 #endif

arch/ibm/ibm.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message IbmDecoderProto {
 	// Next: 11

arch/macintosh/decoder.cc

@@ -1,33 +1,37 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "macintosh.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/macintosh/macintosh.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
 const FluxPattern SECTOR_RECORD_PATTERN(24, MAC_SECTOR_RECORD);
 const FluxPattern DATA_RECORD_PATTERN(24, MAC_DATA_RECORD);
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
     {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "data_gcr.h"
+#undef GCR_ENTRY
     }
     return -1;
 }
 
-/* This is extremely inspired by the MESS implementation, written by Nathan Woods
- * and R. Belmont: https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap_dsk35.cpp
+/* This is extremely inspired by the MESS implementation, written by Nathan
+ * Woods and R. Belmont:
+ * https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap_dsk35.cpp
  */
 static Bytes decode_crazy_data(const Bytes& input, Sector::Status& status)
 {
@@ -41,7 +45,7 @@ static Bytes decode_crazy_data(const Bytes& input, Sector::Status& status)
     uint8_t b2[LOOKUP_LEN + 1];
     uint8_t b3[LOOKUP_LEN + 1];
 
-    for (int i=0; i<=LOOKUP_LEN; i++)
+    for (int i = 0; i <= LOOKUP_LEN; i++)
     {
         uint8_t w4 = br.read_8();
         uint8_t w1 = br.read_8();
@@ -125,67 +129,68 @@ uint8_t decode_side(uint8_t side)
 class MacintoshDecoder : public Decoder
 {
 public:
-	MacintoshDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    MacintoshDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		if (readRaw24() != MAC_SECTOR_RECORD)
-			return;
+    {
+        if (readRaw24() != MAC_SECTOR_RECORD)
+            return;
 
-		/* Read header. */
+        /* Read header. */
 
-		auto header = toBytes(readRawBits(7*8)).slice(0, 7);
-					
-		uint8_t encodedTrack = decode_data_gcr(header[0]);
-		if (encodedTrack != (_sector->physicalTrack & 0x3f))
-			return;
-					
-		uint8_t encodedSector = decode_data_gcr(header[1]);
-		uint8_t encodedSide = decode_data_gcr(header[2]);
-		uint8_t formatByte = decode_data_gcr(header[3]);
-		uint8_t wantedsum = decode_data_gcr(header[4]);
+        auto header = toBytes(readRawBits(7 * 8)).slice(0, 7);
 
-		if (encodedSector > 11)
-			return;
+        uint8_t encodedTrack = decode_data_gcr(header[0]);
+        if (encodedTrack != (_sector->physicalTrack & 0x3f))
+            return;
 
-		_sector->logicalTrack = _sector->physicalTrack;
-		_sector->logicalSide = decode_side(encodedSide);
-		_sector->logicalSector = encodedSector;
-		uint8_t gotsum = (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;
-		if (wantedsum == gotsum)
-			_sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
-	}
+        uint8_t encodedSector = decode_data_gcr(header[1]);
+        uint8_t encodedSide = decode_data_gcr(header[2]);
+        uint8_t formatByte = decode_data_gcr(header[3]);
+        uint8_t wantedsum = decode_data_gcr(header[4]);
+
+        if (encodedSector > 11)
+            return;
+
+        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalSide = decode_side(encodedSide);
+        _sector->logicalSector = encodedSector;
+        uint8_t gotsum =
+            (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;
+        if (wantedsum == gotsum)
+            _sector->status =
+                Sector::DATA_MISSING; /* unintuitive but correct */
+    }
 
     void decodeDataRecord() override
-	{
-		if (readRaw24() != MAC_DATA_RECORD)
-			return;
+    {
+        if (readRaw24() != MAC_DATA_RECORD)
+            return;
 
-		/* Read data. */
+        /* Read data. */
 
-		readRawBits(8); /* skip spare byte */
-		auto inputbuffer = toBytes(readRawBits(MAC_ENCODED_SECTOR_LENGTH*8))
-			.slice(0, MAC_ENCODED_SECTOR_LENGTH);
+        readRawBits(8); /* skip spare byte */
+        auto inputbuffer = toBytes(readRawBits(MAC_ENCODED_SECTOR_LENGTH * 8))
+                               .slice(0, MAC_ENCODED_SECTOR_LENGTH);
 
-		for (unsigned i=0; i<inputbuffer.size(); i++)
-			inputbuffer[i] = decode_data_gcr(inputbuffer[i]);
-			
-		_sector->status = Sector::BAD_CHECKSUM;
-		Bytes userData = decode_crazy_data(inputbuffer, _sector->status);
-		_sector->data.clear();
-		_sector->data.writer().append(userData.slice(12, 512)).append(userData.slice(0, 12));
-	}
+        for (unsigned i = 0; i < inputbuffer.size(); i++)
+            inputbuffer[i] = decode_data_gcr(inputbuffer[i]);
+
+        _sector->status = Sector::BAD_CHECKSUM;
+        Bytes userData = decode_crazy_data(inputbuffer, _sector->status);
+        _sector->data.clear();
+        _sector->data.writer()
+            .append(userData.slice(12, 512))
+            .append(userData.slice(0, 12));
+    }
 };
 
 std::unique_ptr<Decoder> createMacintoshDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new MacintoshDecoder(config));
+    return std::unique_ptr<Decoder>(new MacintoshDecoder(config));
 }
-

arch/macintosh/encoder.cc

@@ -1,13 +1,13 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "macintosh.h"
-#include "crc.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/macintosh/macintosh.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
 #include "fmt/format.h"
 #include "lib/encoders/encoders.pb.h"
-#include "lib/layout.h"
+#include "lib/data/layout.h"
 #include "arch/macintosh/macintosh.pb.h"
 #include <ctype.h>
 
@@ -16,14 +16,14 @@ static bool lastBit;
 static double clockRateUsForTrack(unsigned track)
 {
     if (track < 16)
-        return 2.623;
+        return 2.63;
     if (track < 32)
-        return 2.861;
+        return 2.89;
     if (track < 48)
-        return 3.148;
+        return 3.20;
     if (track < 64)
-        return 3.497;
-    return 3.934;
+        return 3.57;
+    return 3.98;
 }
 
 static unsigned sectorsForTrack(unsigned track)
@@ -174,7 +174,7 @@ static void write_sector(std::vector<bool>& bits,
     const std::shared_ptr<const Sector>& sector)
 {
     if ((sector->data.size() != 512) && (sector->data.size() != 524))
-        Error() << "unsupported sector size --- you must pick 512 or 524";
+        error("unsupported sector size --- you must pick 512 or 524");
 
     write_bits(bits, cursor, 0xff, 1 * 8); /* pad byte */
     for (int i = 0; i < 7; i++)
@@ -239,13 +239,12 @@ public:
             write_sector(bits, cursor, sector);
 
         if (cursor >= bits.size())
-            Error() << fmt::format(
-                "track data overrun by {} bits", cursor - bits.size());
+            error("track data overrun by {} bits", cursor - bits.size());
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
-        fluxmap->appendBits(bits,
-            calculatePhysicalClockPeriod(clockRateUs * 1e3, 200e6));
+        fluxmap->appendBits(
+            bits, calculatePhysicalClockPeriod(clockRateUs * 1e3, 200e6));
         return fluxmap;
     }
 
@@ -253,8 +252,7 @@ private:
     const MacintoshEncoderProto& _config;
 };
 
-std::unique_ptr<Encoder> createMacintoshEncoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createMacintoshEncoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new MacintoshEncoder(config));
 }

arch/macintosh/macintosh.h

@@ -1,12 +1,12 @@
 #ifndef MACINTOSH_H
 #define MACINTOSH_H
 
-#define MAC_SECTOR_RECORD   0xd5aa96 /* 1101 0101 1010 1010 1001 0110 */
-#define MAC_DATA_RECORD     0xd5aaad /* 1101 0101 1010 1010 1010 1101 */
+#define MAC_SECTOR_RECORD 0xd5aa96 /* 1101 0101 1010 1010 1001 0110 */
+#define MAC_DATA_RECORD 0xd5aaad   /* 1101 0101 1010 1010 1010 1101 */
 
-#define MAC_SECTOR_LENGTH   524 /* yes, really */
+#define MAC_SECTOR_LENGTH 524 /* yes, really */
 #define MAC_ENCODED_SECTOR_LENGTH 703
-#define MAC_FORMAT_BYTE     0x22
+#define MAC_FORMAT_BYTE 0x22
 
 #define MAC_TRACKS_PER_DISK 80
 
@@ -15,8 +15,9 @@ class Decoder;
 class DecoderProto;
 class EncoderProto;
 
-extern std::unique_ptr<Decoder> createMacintoshDecoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createMacintoshEncoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createMacintoshDecoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createMacintoshEncoder(
+    const EncoderProto& config);
 
 #endif
-

arch/macintosh/macintosh.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message MacintoshDecoderProto {}
 

arch/micropolis/decoder.cc

@@ -1,10 +1,11 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "micropolis.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/micropolis/micropolis.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include "lib/decoders/decoders.pb.h"
 
@@ -20,17 +21,20 @@ static const FluxPattern SECTOR_SYNC_PATTERN(64, 0xAAAAAAAAAAAA5555LL);
 static const FluxPattern SECTOR_ADVANCE_PATTERN(64, 0xAAAAAAAAAAAAAAAALL);
 
 /* Standard Micropolis checksum.  Adds all bytes, with carry. */
-uint8_t micropolisChecksum(const Bytes& bytes) {
-	ByteReader br(bytes);
-	uint16_t sum = 0;
-	while (!br.eof()) {
-		if (sum > 0xFF) {
-			sum -= 0x100 - 1;
-		}
-		sum += br.read_8();
-	}
-	/* The last carry is ignored */
-	return sum & 0xFF;
+uint8_t micropolisChecksum(const Bytes& bytes)
+{
+    ByteReader br(bytes);
+    uint16_t sum = 0;
+    while (!br.eof())
+    {
+        if (sum > 0xFF)
+        {
+            sum -= 0x100 - 1;
+        }
+        sum += br.read_8();
+    }
+    /* The last carry is ignored */
+    return sum & 0xFF;
 }
 
 /* Vector MZOS does not use the standard Micropolis checksum.
@@ -41,145 +45,251 @@ uint8_t micropolisChecksum(const Bytes& bytes) {
  * Unlike the Micropolis checksum, this does not cover the 12-byte
  * header (track, sector, 10 OS-specific bytes.)
  */
-uint8_t mzosChecksum(const Bytes& bytes) {
-	ByteReader br(bytes);
-	uint8_t checksum = 0;
-	uint8_t databyte;
+uint8_t mzosChecksum(const Bytes& bytes)
+{
+    ByteReader br(bytes);
+    uint8_t checksum = 0;
+    uint8_t databyte;
 
-	while (!br.eof()) {
-		databyte = br.read_8();
-	checksum ^= ((databyte << 1) | (databyte >> 7));
-	}
+    while (!br.eof())
+    {
+        databyte = br.read_8();
+        checksum ^= ((databyte << 1) | (databyte >> 7));
+    }
 
-	return checksum;
+    return checksum;
+}
+
+static uint8_t b(uint32_t field, uint8_t pos)
+{
+    return (field >> pos) & 1;
+}
+
+static uint8_t eccNextBit(uint32_t ecc, uint8_t data_bit)
+{
+    // This is 0x81932080 which is 0x0104C981 with reversed bits
+    return b(ecc, 7) ^ b(ecc, 13) ^ b(ecc, 16) ^ b(ecc, 17) ^ b(ecc, 20) ^
+           b(ecc, 23) ^ b(ecc, 24) ^ b(ecc, 31) ^ data_bit;
+}
+
+uint32_t vectorGraphicEcc(const Bytes& bytes)
+{
+    uint32_t e = 0;
+    Bytes payloadBytes = bytes.slice(0, bytes.size() - 4);
+    ByteReader payload(payloadBytes);
+    while (!payload.eof())
+    {
+        uint8_t byte = payload.read_8();
+        for (int i = 0; i < 8; i++)
+        {
+            e = (e << 1) | eccNextBit(e, byte >> 7);
+            byte <<= 1;
+        }
+    }
+    Bytes trailerBytes = bytes.slice(bytes.size() - 4);
+    ByteReader trailer(trailerBytes);
+    uint32_t res = e;
+    while (!trailer.eof())
+    {
+        uint8_t byte = trailer.read_8();
+        for (int i = 0; i < 8; i++)
+        {
+            res = (res << 1) | eccNextBit(e, byte >> 7);
+            e <<= 1;
+            byte <<= 1;
+        }
+    }
+    return res;
+}
+
+/* Fixes bytes when possible, returning true if changed. */
+static bool vectorGraphicEccFix(Bytes& bytes, uint32_t syndrome)
+{
+    uint32_t ecc = syndrome;
+    int pos = (MICROPOLIS_ENCODED_SECTOR_SIZE - 5) * 8 + 7;
+    bool aligned = false;
+    while ((ecc & 0xff000000) == 0)
+    {
+        pos += 8;
+        ecc <<= 8;
+    }
+    for (; pos >= 0; pos--)
+    {
+        bool bit = ecc & 1;
+        ecc >>= 1;
+        if (bit)
+            ecc ^= 0x808264c0;
+        if ((ecc & 0xff07ffff) == 0)
+            aligned = true;
+        if (aligned && pos % 8 == 0)
+            break;
+    }
+    if (pos < 0)
+        return false;
+    bytes[pos / 8] ^= ecc >> 16;
+    return true;
 }
 
 class MicropolisDecoder : public Decoder
 {
 public:
-	MicropolisDecoder(const DecoderProto& config):
-		Decoder(config),
-		_config(config.micropolis())
-	{
-		_checksumType = _config.checksum_type();
-	}
+    MicropolisDecoder(const DecoderProto& config):
+        Decoder(config),
+        _config(config.micropolis())
+    {
+        _checksumType = _config.checksum_type();
+    }
 
-	nanoseconds_t advanceToNextRecord() override
-	{
-		nanoseconds_t now = tell().ns();
+    nanoseconds_t advanceToNextRecord() override
+    {
+        nanoseconds_t now = tell().ns();
 
-		/* For all but the first sector, seek to the next sector pulse.
-		 * The first sector does not contain the sector pulse in the fluxmap.
-		 */
-		if (now != 0) {
-			seekToIndexMark();
-			now = tell().ns();
-		}
+        /* For all but the first sector, seek to the next sector pulse.
+         * The first sector does not contain the sector pulse in the fluxmap.
+         */
+        if (now != 0)
+        {
+            seekToIndexMark();
+            now = tell().ns();
+        }
 
-		/* Discard a possible partial sector at the end of the track.
-		 * This partial sector could be mistaken for a conflicted sector, if
-		 * whatever data read happens to match the checksum of 0, which is
-		 * rare, but has been observed on some disks.
-		 */
-		if (now > (getFluxmapDuration() - 12.5e6)) {
-			seekToIndexMark();
-			return 0;
-		}
+        /* Discard a possible partial sector at the end of the track.
+         * This partial sector could be mistaken for a conflicted sector, if
+         * whatever data read happens to match the checksum of 0, which is
+         * rare, but has been observed on some disks. There's 570uS of slack in
+         * each sector, after accounting for preamble, data, and postamble.
+         */
+        if (now > (getFluxmapDuration() - 12.0e6))
+        {
+            seekToIndexMark();
+            return 0;
+        }
 
-		nanoseconds_t clock = seekToPattern(SECTOR_SYNC_PATTERN);
+        nanoseconds_t clock = seekToPattern(SECTOR_SYNC_PATTERN);
 
-		auto syncDelta = tell().ns() - now;
-		/* Due to the weak nature of the Micropolis SYNC patern,
-		 * it's possible to detect a false SYNC during the gap
-		 * between the sector pulse and the write gate.  If the SYNC
-		 * is detected less than 100uS after the sector pulse, search
-		 * for another valid SYNC.
-		 *
-		 * Reference: Vector Micropolis Disk Controller Board Technical
-		 * Information Manual, pp. 1-16.
-		 */
-		if ((syncDelta > 0) && (syncDelta < 100e3)) {
-			seekToPattern(SECTOR_ADVANCE_PATTERN);
-			clock = seekToPattern(SECTOR_SYNC_PATTERN);
-		}
+        auto syncDelta = tell().ns() - now;
+        /* Due to the weak nature of the Micropolis SYNC patern,
+         * it's possible to detect a false SYNC during the gap
+         * between the sector pulse and the write gate.  If the SYNC
+         * is detected less than 100uS after the sector pulse, search
+         * for another valid SYNC.
+         *
+         * Reference: Vector Micropolis Disk Controller Board Technical
+         * Information Manual, pp. 1-16.
+         */
+        if ((syncDelta > 0) && (syncDelta < 100e3))
+        {
+            seekToPattern(SECTOR_ADVANCE_PATTERN);
+            clock = seekToPattern(SECTOR_SYNC_PATTERN);
+        }
 
-		_sector->headerStartTime = tell().ns();
+        _sector->headerStartTime = tell().ns();
 
-		/* seekToPattern() can skip past the index hole, if this happens
-		 * too close to the end of the Fluxmap, discard the sector.
-		 */
-		if (_sector->headerStartTime > (getFluxmapDuration() - 12.5e6)) {
-			return 0;
-		}
+        /* seekToPattern() can skip past the index hole, if this happens
+         * too close to the end of the Fluxmap, discard the sector. The
+         * preamble was expected to be 640uS long.
+         */
+        if (_sector->headerStartTime > (getFluxmapDuration() - 11.3e6))
+        {
+            return 0;
+        }
 
-		return clock;
-	}
+        return clock;
+    }
 
-	void decodeSectorRecord() override
-	{
-		readRawBits(48);
-		auto rawbits = readRawBits(MICROPOLIS_ENCODED_SECTOR_SIZE*16);
-		auto bytes = decodeFmMfm(rawbits).slice(0, MICROPOLIS_ENCODED_SECTOR_SIZE);
-		ByteReader br(bytes);
+    void decodeSectorRecord() override
+    {
+        readRawBits(48);
+        auto rawbits = readRawBits(MICROPOLIS_ENCODED_SECTOR_SIZE * 16);
+        auto bytes =
+            decodeFmMfm(rawbits).slice(0, MICROPOLIS_ENCODED_SECTOR_SIZE);
 
-		int syncByte = br.read_8();  /* sync */
-		if (syncByte != 0xFF)
-			return;
+        bool eccPresent = bytes[274] == 0xaa;
+        uint32_t ecc = 0;
+        if (_config.ecc_type() == MicropolisDecoderProto::VECTOR && eccPresent)
+        {
+            ecc = vectorGraphicEcc(bytes.slice(0, 274));
+            if (ecc != 0)
+            {
+                vectorGraphicEccFix(bytes, ecc);
+                ecc = vectorGraphicEcc(bytes.slice(0, 274));
+            }
+        }
 
-		_sector->logicalTrack = br.read_8();
-		_sector->logicalSide = _sector->physicalSide;
-		_sector->logicalSector = br.read_8();
-		if (_sector->logicalSector > 15)
-			return;
-		if (_sector->logicalTrack > 76)
-			return;
-		if (_sector->logicalTrack != _sector->physicalTrack)
-			return;
+        ByteReader br(bytes);
 
-		br.read(10);  /* OS data or padding */
-		auto data = br.read(MICROPOLIS_PAYLOAD_SIZE);
-		uint8_t wantChecksum = br.read_8();
+        int syncByte = br.read_8(); /* sync */
+        if (syncByte != 0xFF)
+            return;
 
-		/* If not specified, automatically determine the checksum type.
-		* Once the checksum type is determined, it will be used for the
-		* entire disk.
-		*/
-		if (_checksumType == MicropolisDecoderProto::AUTO) {
-			/* Calculate both standard Micropolis (MDOS, CP/M, OASIS) and MZOS checksums */
-			if (wantChecksum == micropolisChecksum(bytes.slice(1, 2+266))) {
-				_checksumType = MicropolisDecoderProto::MICROPOLIS;
-			} else if (wantChecksum == mzosChecksum(bytes.slice(MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE))) {
-				_checksumType = MicropolisDecoderProto::MZOS;
-				std::cout << "Note: MZOS checksum detected." << std::endl;
-			}
-		}
+        _sector->logicalTrack = br.read_8();
+        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalSector = br.read_8();
+        if (_sector->logicalSector > 15)
+            return;
+        if (_sector->logicalTrack > 76)
+            return;
+        if (_sector->logicalTrack != _sector->physicalTrack)
+            return;
 
-		uint8_t gotChecksum;
+        br.read(10); /* OS data or padding */
+        auto data = br.read(MICROPOLIS_PAYLOAD_SIZE);
+        uint8_t wantChecksum = br.read_8();
 
-		if (_checksumType == MicropolisDecoderProto::MZOS) {
-			gotChecksum = mzosChecksum(bytes.slice(MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE));
-		} else {
-			gotChecksum = micropolisChecksum(bytes.slice(1, 2+266));
-		}
+        /* If not specified, automatically determine the checksum type.
+         * Once the checksum type is determined, it will be used for the
+         * entire disk.
+         */
+        if (_checksumType == MicropolisDecoderProto::AUTO)
+        {
+            /* Calculate both standard Micropolis (MDOS, CP/M, OASIS) and MZOS
+             * checksums */
+            if (wantChecksum == micropolisChecksum(bytes.slice(1, 2 + 266)))
+            {
+                _checksumType = MicropolisDecoderProto::MICROPOLIS;
+            }
+            else if (wantChecksum ==
+                     mzosChecksum(bytes.slice(
+                         MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE)))
+            {
+                _checksumType = MicropolisDecoderProto::MZOS;
+                std::cout << "Note: MZOS checksum detected." << std::endl;
+            }
+        }
 
-		br.read(5);  /* 4 byte ECC and ECC-present flag */
+        uint8_t gotChecksum;
 
-		if (_config.sector_output_size() == MICROPOLIS_PAYLOAD_SIZE)
-			_sector->data = data;
-		else if (_config.sector_output_size() == MICROPOLIS_ENCODED_SECTOR_SIZE)
-			_sector->data = bytes;
-		else
-			Error() << "Sector output size may only be 256 or 275";
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        if (_checksumType == MicropolisDecoderProto::MZOS)
+        {
+            gotChecksum = mzosChecksum(
+                bytes.slice(MICROPOLIS_HEADER_SIZE, MICROPOLIS_PAYLOAD_SIZE));
+        }
+        else
+        {
+            gotChecksum = micropolisChecksum(bytes.slice(1, 2 + 266));
+        }
+
+        br.read(5); /* 4 byte ECC and ECC-present flag */
+
+        if (_config.sector_output_size() == MICROPOLIS_PAYLOAD_SIZE)
+            _sector->data = data;
+        else if (_config.sector_output_size() == MICROPOLIS_ENCODED_SECTOR_SIZE)
+            _sector->data = bytes;
+        else
+            error("Sector output size may only be 256 or 275");
+        if (wantChecksum == gotChecksum && (!eccPresent || ecc == 0))
+            _sector->status = Sector::OK;
+        else
+            _sector->status = Sector::BAD_CHECKSUM;
+    }
 
 private:
-	const MicropolisDecoderProto& _config;
-	MicropolisDecoderProto_ChecksumType _checksumType;	/* -1 = auto, 1 = Micropolis, 2=MZOS */
+    const MicropolisDecoderProto& _config;
+    MicropolisDecoderProto_ChecksumType
+        _checksumType; /* -1 = auto, 1 = Micropolis, 2=MZOS */
 };
 
 std::unique_ptr<Decoder> createMicropolisDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new MicropolisDecoder(config));
+    return std::unique_ptr<Decoder>(new MicropolisDecoder(config));
 }
-

arch/micropolis/encoder.cc

@@ -1,18 +1,19 @@
-#include "globals.h"
-#include "micropolis.h"
-#include "sector.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "arch/micropolis/micropolis.h"
+#include "lib/data/sector.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "lib/data/image.h"
 #include "lib/encoders/encoders.pb.h"
 
 static void write_sector(std::vector<bool>& bits,
     unsigned& cursor,
-    const std::shared_ptr<const Sector>& sector)
+    const std::shared_ptr<const Sector>& sector,
+    MicropolisEncoderProto::EccType eccType)
 {
     if ((sector->data.size() != 256) &&
         (sector->data.size() != MICROPOLIS_ENCODED_SECTOR_SIZE))
-        Error() << "unsupported sector size --- you must pick 256 or 275";
+        error("unsupported sector size --- you must pick 256 or 275");
 
     int fullSectorSize = 40 + MICROPOLIS_ENCODED_SECTOR_SIZE + 40 + 35;
     auto fullSector = std::make_shared<std::vector<uint8_t>>();
@@ -24,8 +25,9 @@ static void write_sector(std::vector<bool>& bits,
     if (sector->data.size() == MICROPOLIS_ENCODED_SECTOR_SIZE)
     {
         if (sector->data[0] != 0xFF)
-            Error() << "275 byte sector doesn't start with sync byte 0xFF. "
-                       "Corrupted sector";
+            error(
+                "275 byte sector doesn't start with sync byte 0xFF. "
+                "Corrupted sector");
         uint8_t wantChecksum = sector->data[1 + 2 + 266];
         uint8_t gotChecksum =
             micropolisChecksum(sector->data.slice(1, 2 + 266));
@@ -44,8 +46,16 @@ static void write_sector(std::vector<bool>& bits,
             writer.write_8(0); /* Padding */
         writer += sector->data;
         writer.write_8(micropolisChecksum(sectorData.slice(1)));
-        for (int i = 0; i < 5; i++)
-            writer.write_8(0); /* 4 byte ECC and ECC not present flag */
+
+        uint8_t eccPresent = 0;
+        uint32_t ecc = 0;
+        if (eccType == MicropolisEncoderProto::VECTOR)
+        {
+            eccPresent = 0xaa;
+            ecc = vectorGraphicEcc(sectorData + Bytes(4));
+        }
+        writer.write_be32(ecc);
+        writer.write_8(eccPresent);
     }
     for (uint8_t b : sectorData)
         fullSector->push_back(b);
@@ -57,7 +67,7 @@ static void write_sector(std::vector<bool>& bits,
         fullSector->push_back(0);
 
     if (fullSector->size() != fullSectorSize)
-        Error() << "sector mismatched length";
+        error("sector mismatched length");
     bool lastBit = false;
     encodeMfm(bits, cursor, fullSector, lastBit);
     /* filler */
@@ -85,19 +95,34 @@ public:
             (_config.rotational_period_ms() * 1e3) / _config.clock_period_us();
 
         std::vector<bool> bits(bitsPerRevolution);
+        std::vector<unsigned> indexes;
+        unsigned prev_cursor = 0;
         unsigned cursor = 0;
 
         for (const auto& sectorData : sectors)
-            write_sector(bits, cursor, sectorData);
+        {
+            indexes.push_back(cursor);
+            prev_cursor = cursor;
+            write_sector(bits, cursor, sectorData, _config.ecc_type());
+        }
+        indexes.push_back(prev_cursor + (cursor - prev_cursor) / 2);
+        indexes.push_back(cursor);
 
         if (cursor != bits.size())
-            Error() << "track data mismatched length";
+            error("track data mismatched length");
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
-        fluxmap->appendBits(bits,
-            calculatePhysicalClockPeriod(
-                _config.clock_period_us() * 1e3,
-                _config.rotational_period_ms() * 1e6));
+        nanoseconds_t clockPeriod =
+            calculatePhysicalClockPeriod(_config.clock_period_us() * 1e3,
+                _config.rotational_period_ms() * 1e6);
+        auto pos = bits.begin();
+        for (int i = 1; i < indexes.size(); i++)
+        {
+            auto end = bits.begin() + indexes[i];
+            fluxmap->appendBits(std::vector<bool>(pos, end), clockPeriod);
+            fluxmap->appendIndex();
+            pos = end;
+        }
         return fluxmap;
     }
 
@@ -105,8 +130,7 @@ private:
     const MicropolisEncoderProto& _config;
 };
 
-std::unique_ptr<Encoder> createMicropolisEncoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createMicropolisEncoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new MicropolisEncoder(config));
 }

arch/micropolis/micropolis.h

@@ -1,18 +1,22 @@
 #ifndef MICROPOLIS_H
 #define MICROPOLIS_H
 
-#define MICROPOLIS_PAYLOAD_SIZE         (256)
-#define MICROPOLIS_HEADER_SIZE          (1+2+10)
-#define MICROPOLIS_ENCODED_SECTOR_SIZE  (MICROPOLIS_HEADER_SIZE + MICROPOLIS_PAYLOAD_SIZE + 6)
+#define MICROPOLIS_PAYLOAD_SIZE (256)
+#define MICROPOLIS_HEADER_SIZE (1 + 2 + 10)
+#define MICROPOLIS_ENCODED_SECTOR_SIZE \
+    (MICROPOLIS_HEADER_SIZE + MICROPOLIS_PAYLOAD_SIZE + 6)
 
 class Decoder;
 class Encoder;
 class EncoderProto;
 class DecoderProto;
 
-extern std::unique_ptr<Decoder> createMicropolisDecoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createMicropolisEncoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createMicropolisDecoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createMicropolisEncoder(
+    const EncoderProto& config);
 
 extern uint8_t micropolisChecksum(const Bytes& bytes);
+extern uint32_t vectorGraphicEcc(const Bytes& bytes);
 
 #endif

arch/micropolis/micropolis.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message MicropolisDecoderProto {
 	enum ChecksumType {
@@ -8,17 +8,30 @@ message MicropolisDecoderProto {
 		MICROPOLIS = 1;
 		MZOS = 2;
 	}
+	enum EccType {
+		NONE = 0;
+		VECTOR = 1;
+	}
 
 	optional int32 sector_output_size = 1 [default = 256,
 		(help) = "How much of the raw sector should be saved. Must be 256 or 275"];
 	optional ChecksumType checksum_type = 2 [default = AUTO,
 		(help) = "Checksum type to use: AUTO, MICROPOLIS, MZOS"];
+	optional EccType ecc_type = 3 [default = NONE,
+		(help) = "ECC type to use: NONE, VECTOR"];
 }
 
 message MicropolisEncoderProto {
+    enum EccType {
+        NONE = 0;
+        VECTOR = 1;
+    }
+
     optional double clock_period_us = 1
         [ default = 2.0, (help) = "clock rate on the real device" ];
     optional double rotational_period_ms = 2
-        [ default = 166.0, (help) = "rotational period on the real device" ];
+        [ default = 200.0, (help) = "rotational period on the real device" ];
+    optional EccType ecc_type = 3 [default = NONE,
+        (help) = "ECC type to use for IMG data: NONE, VECTOR"];
 }
 

arch/mx/decoder.cc

@@ -1,10 +1,11 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "mx/mx.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "sector.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "arch/mx/mx.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
 #include <string.h>
 
 const int SECTOR_SIZE = 256;
@@ -26,52 +27,51 @@ const FluxPattern ID_PATTERN(32, 0xaaaaffaf);
 class MxDecoder : public Decoder
 {
 public:
-	MxDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    MxDecoder(const DecoderProto& config): Decoder(config) {}
 
     void beginTrack() override
-	{
-		_clock = _sector->clock = seekToPattern(ID_PATTERN);
-		_currentSector = 0;
-	}
+    {
+        _clock = _sector->clock = seekToPattern(ID_PATTERN);
+        _currentSector = 0;
+    }
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		if (_currentSector == 11)
-		{
-			/* That was the last sector on the disk. */
-			return 0;
-		}
-		else
-			return _clock;
-	}
+    {
+        if (_currentSector == 11)
+        {
+            /* That was the last sector on the disk. */
+            return 0;
+        }
+        else
+            return _clock;
+    }
 
     void decodeSectorRecord() override
-	{
-		/* Skip the ID pattern and track word, which is only present on the
-		 * first sector. We don't trust the track word because some driver
-		 * don't write it correctly. */
+    {
+        /* Skip the ID pattern and track word, which is only present on the
+         * first sector. We don't trust the track word because some driver
+         * don't write it correctly. */
 
-		if (_currentSector == 0)
-			readRawBits(64);
+        if (_currentSector == 0)
+            readRawBits(64);
 
-		auto bits = readRawBits((SECTOR_SIZE+2)*16);
-		auto bytes = decodeFmMfm(bits).slice(0, SECTOR_SIZE+2);
+        auto bits = readRawBits((SECTOR_SIZE + 2) * 16);
+        auto bytes = decodeFmMfm(bits).slice(0, SECTOR_SIZE + 2);
 
-		uint16_t gotChecksum = 0;
-		ByteReader br(bytes);
-		for (int i=0; i<(SECTOR_SIZE/2); i++)
-			gotChecksum += br.read_be16();
-		uint16_t wantChecksum = br.read_be16();
+        uint16_t gotChecksum = 0;
+        ByteReader br(bytes);
+        for (int i = 0; i < (SECTOR_SIZE / 2); i++)
+            gotChecksum += br.read_be16();
+        uint16_t wantChecksum = br.read_be16();
 
-		_sector->logicalTrack = _sector->physicalTrack;
-		_sector->logicalSide = _sector->physicalSide;
-		_sector->logicalSector = _currentSector;
-		_sector->data = bytes.slice(0, SECTOR_SIZE).swab();
-		_sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-		_currentSector++;
-	}
+        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalSector = _currentSector;
+        _sector->data = bytes.slice(0, SECTOR_SIZE).swab();
+        _sector->status =
+            (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+        _currentSector++;
+    }
 
 private:
     nanoseconds_t _clock;
@@ -80,7 +80,5 @@ private:
 
 std::unique_ptr<Decoder> createMxDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new MxDecoder(config));
+    return std::unique_ptr<Decoder>(new MxDecoder(config));
 }
-
-

arch/mx/mx.h

@@ -1,7 +1,7 @@
 #ifndef MX_H
 #define MX_H
 
-#include "decoders/decoders.h"
+#include "lib/decoders/decoders.h"
 
 extern std::unique_ptr<Decoder> createMxDecoder(const DecoderProto& config);
 

arch/northstar/decoder.cc

@@ -11,18 +11,19 @@
  * sure that the hardSectorId is correct.
  */
 
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "northstar.h"
-#include "bytes.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/northstar/northstar.h"
+#include "lib/core/bytes.h"
 #include "lib/decoders/decoders.pb.h"
 #include "fmt/format.h"
 
 #define MFM_ID 0xaaaaaaaaaaaa5545LL
-#define FM_ID  0xaaaaaaaaaaaaffefLL
+#define FM_ID 0xaaaaaaaaaaaaffefLL
 /*
  * MFM sectors have 32 bytes of 00's followed by two sync characters,
  * specified in the North Star MDS manual as 0xFBFB.
@@ -44,133 +45,143 @@ static const FluxPattern MFM_PATTERN(64, MFM_ID);
  */
 static const FluxPattern FM_PATTERN(64, FM_ID);
 
-const FluxMatchers ANY_SECTOR_PATTERN(
-	{
-		&MFM_PATTERN,
-		&FM_PATTERN,
-	}
-);
+const FluxMatchers ANY_SECTOR_PATTERN({
+    &MFM_PATTERN,
+    &FM_PATTERN,
+});
 
 /* Checksum is initially 0.
  * For each data byte, XOR with the current checksum.
  * Rotate checksum left, carrying bit 7 to bit 0.
  */
-uint8_t northstarChecksum(const Bytes& bytes) {
-	ByteReader br(bytes);
-	uint8_t checksum = 0;
+uint8_t northstarChecksum(const Bytes& bytes)
+{
+    ByteReader br(bytes);
+    uint8_t checksum = 0;
 
-	while (!br.eof()) {
-		checksum ^= br.read_8();
-		checksum = ((checksum << 1) | ((checksum >> 7)));
-	}
+    while (!br.eof())
+    {
+        checksum ^= br.read_8();
+        checksum = ((checksum << 1) | ((checksum >> 7)));
+    }
 
-	return checksum;
+    return checksum;
 }
 
 class NorthstarDecoder : public Decoder
 {
 public:
-	NorthstarDecoder(const DecoderProto& config):
-		Decoder(config),
-		_config(config.northstar())
-	{}
+    NorthstarDecoder(const DecoderProto& config):
+        Decoder(config),
+        _config(config.northstar())
+    {
+    }
 
-	/* Search for FM or MFM sector record */
-	nanoseconds_t advanceToNextRecord() override
-	{
-		nanoseconds_t now = tell().ns();
+    /* Search for FM or MFM sector record */
+    nanoseconds_t advanceToNextRecord() override
+    {
+        nanoseconds_t now = tell().ns();
 
-		/* For all but the first sector, seek to the next sector pulse.
-		 * The first sector does not contain the sector pulse in the fluxmap.
-		 */
-		if (now != 0) {
-			seekToIndexMark();
-			now = tell().ns();
-		}
+        /* For all but the first sector, seek to the next sector pulse.
+         * The first sector does not contain the sector pulse in the fluxmap.
+         */
+        if (now != 0)
+        {
+            seekToIndexMark();
+            now = tell().ns();
+        }
 
-		/* Discard a possible partial sector at the end of the track.
-		 * This partial sector could be mistaken for a conflicted sector, if
-		 * whatever data read happens to match the checksum of 0, which is
-		 * rare, but has been observed on some disks.
-		 */
-		if (now > (getFluxmapDuration() - 21e6)) {
-			seekToIndexMark();
-			return 0;
-		}
+        /* Discard a possible partial sector at the end of the track.
+         * This partial sector could be mistaken for a conflicted sector, if
+         * whatever data read happens to match the checksum of 0, which is
+         * rare, but has been observed on some disks.
+         */
+        if (now > (getFluxmapDuration() - 21e6))
+        {
+            seekToIndexMark();
+            return 0;
+        }
 
-		int msSinceIndex = std::round(now / 1e6);
+        int msSinceIndex = std::round(now / 1e6);
 
-		/* Note that the seekToPattern ignores the sector pulses, so if
-		 * a sector is not found for some reason, the seek will advance
-		 * past one or more sector pulses.  For this reason, calculate
-		 * _hardSectorId after the sector header is found.
-		 */
-		nanoseconds_t clock = seekToPattern(ANY_SECTOR_PATTERN);
-		_sector->headerStartTime = tell().ns();
+        /* Note that the seekToPattern ignores the sector pulses, so if
+         * a sector is not found for some reason, the seek will advance
+         * past one or more sector pulses.  For this reason, calculate
+         * _hardSectorId after the sector header is found.
+         */
+        nanoseconds_t clock = seekToPattern(ANY_SECTOR_PATTERN);
+        _sector->headerStartTime = tell().ns();
 
-		/* Discard a possible partial sector. */
-		if (_sector->headerStartTime > (getFluxmapDuration() - 21e6)) {
-			return 0;
-		}
+        /* Discard a possible partial sector. */
+        if (_sector->headerStartTime > (getFluxmapDuration() - 21e6))
+        {
+            return 0;
+        }
 
-		int sectorFoundTimeRaw = std::round(_sector->headerStartTime / 1e6);
-		int sectorFoundTime;
+        int sectorFoundTimeRaw = std::round(_sector->headerStartTime / 1e6);
+        int sectorFoundTime;
 
-		/* Round time to the nearest 20ms */
-		if ((sectorFoundTimeRaw % 20) < 10) {
-			sectorFoundTime = (sectorFoundTimeRaw / 20) * 20;
-		}
-		else {
-			sectorFoundTime = ((sectorFoundTimeRaw + 20) / 20) * 20;
-		}
+        /* Round time to the nearest 20ms */
+        if ((sectorFoundTimeRaw % 20) < 10)
+        {
+            sectorFoundTime = (sectorFoundTimeRaw / 20) * 20;
+        }
+        else
+        {
+            sectorFoundTime = ((sectorFoundTimeRaw + 20) / 20) * 20;
+        }
 
-		/* Calculate the sector ID based on time since the index */
-		_hardSectorId = (sectorFoundTime / 20) % 10;
+        /* Calculate the sector ID based on time since the index */
+        _hardSectorId = (sectorFoundTime / 20) % 10;
 
-		return clock;
-	}
+        return clock;
+    }
 
-	void decodeSectorRecord() override
-	{
-		uint64_t id = toBytes(readRawBits(64)).reader().read_be64();
-		unsigned recordSize, payloadSize, headerSize;
+    void decodeSectorRecord() override
+    {
+        uint64_t id = toBytes(readRawBits(64)).reader().read_be64();
+        unsigned recordSize, payloadSize, headerSize;
 
-		if (id == MFM_ID) {
-			recordSize = NORTHSTAR_ENCODED_SECTOR_SIZE_DD;
-			payloadSize = NORTHSTAR_PAYLOAD_SIZE_DD;
-			headerSize = NORTHSTAR_HEADER_SIZE_DD;
-		}
-		else {
-			recordSize = NORTHSTAR_ENCODED_SECTOR_SIZE_SD;
-			payloadSize = NORTHSTAR_PAYLOAD_SIZE_SD;
-			headerSize = NORTHSTAR_HEADER_SIZE_SD;
-		}
+        if (id == MFM_ID)
+        {
+            recordSize = NORTHSTAR_ENCODED_SECTOR_SIZE_DD;
+            payloadSize = NORTHSTAR_PAYLOAD_SIZE_DD;
+            headerSize = NORTHSTAR_HEADER_SIZE_DD;
+        }
+        else
+        {
+            recordSize = NORTHSTAR_ENCODED_SECTOR_SIZE_SD;
+            payloadSize = NORTHSTAR_PAYLOAD_SIZE_SD;
+            headerSize = NORTHSTAR_HEADER_SIZE_SD;
+        }
 
-		auto rawbits = readRawBits(recordSize * 16);
-		auto bytes = decodeFmMfm(rawbits).slice(0, recordSize);
-		ByteReader br(bytes);
+        auto rawbits = readRawBits(recordSize * 16);
+        auto bytes = decodeFmMfm(rawbits).slice(0, recordSize);
+        ByteReader br(bytes);
 
-		_sector->logicalSide = _sector->physicalSide;
-		_sector->logicalSector = _hardSectorId;
-		_sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalSector = _hardSectorId;
+        _sector->logicalTrack = _sector->physicalTrack;
 
-		if (headerSize == NORTHSTAR_HEADER_SIZE_DD) {
-			br.read_8();	/* MFM second Sync char, usually 0xFB */
-		}
+        if (headerSize == NORTHSTAR_HEADER_SIZE_DD)
+        {
+            br.read_8(); /* MFM second Sync char, usually 0xFB */
+        }
 
-		_sector->data = br.read(payloadSize);
-		uint8_t wantChecksum = br.read_8();
-		uint8_t gotChecksum = northstarChecksum(bytes.slice(headerSize - 1, payloadSize));
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = br.read(payloadSize);
+        uint8_t wantChecksum = br.read_8();
+        uint8_t gotChecksum =
+            northstarChecksum(bytes.slice(headerSize - 1, payloadSize));
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 
 private:
-	const NorthstarDecoderProto& _config;
-	uint8_t _hardSectorId;
+    const NorthstarDecoderProto& _config;
+    uint8_t _hardSectorId;
 };
 
 std::unique_ptr<Decoder> createNorthstarDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new NorthstarDecoder(config));
+    return std::unique_ptr<Decoder>(new NorthstarDecoder(config));
 }
-

arch/northstar/encoder.cc

@@ -1,10 +1,10 @@
-#include "globals.h"
-#include "northstar.h"
-#include "sector.h"
-#include "bytes.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "arch/northstar/northstar.h"
+#include "lib/data/sector.h"
+#include "lib/core/bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "lib/data/image.h"
 #include "lib/encoders/encoders.pb.h"
 
 #define GAP_FILL_SIZE_SD 30
@@ -49,7 +49,7 @@ static void write_sector(std::vector<bool>& bits,
             doubleDensity = true;
             break;
         default:
-            Error() << "unsupported sector size --- you must pick 256 or 512";
+            error("unsupported sector size --- you must pick 256 or 512");
             break;
     }
 
@@ -96,9 +96,10 @@ static void write_sector(std::vector<bool>& bits,
             fullSector->push_back(GAP2_FILL_BYTE);
 
         if (fullSector->size() != fullSectorSize)
-            Error() << "sector mismatched length (" << sector->data.size()
-                    << ") expected: " << fullSector->size() << " got "
-                    << fullSectorSize;
+            error("sector mismatched length ({}); expected {}, got {}",
+                sector->data.size(),
+                fullSector->size(),
+                fullSectorSize);
     }
     else
     {
@@ -148,7 +149,7 @@ public:
             write_sector(bits, cursor, sectorData);
 
         if (cursor > bits.size())
-            Error() << "track data overrun";
+            error("track data overrun");
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
         fluxmap->appendBits(bits,
@@ -161,8 +162,7 @@ private:
     const NorthstarEncoderProto& _config;
 };
 
-std::unique_ptr<Encoder> createNorthstarEncoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createNorthstarEncoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new NorthstarEncoder(config));
 }

arch/northstar/northstar.h

@@ -1,7 +1,8 @@
 #ifndef NORTHSTAR_H
 #define NORTHSTAR_H
 
-/* Northstar floppies are 10-hard sectored disks with a sector format as follows:
+/* Northstar floppies are 10-hard sectored disks with a sector format as
+ * follows:
  *
  * |----------------------------------|
  * | SYNC Byte  | Payload  | Checksum |
@@ -12,15 +13,19 @@
  *
  */
 
-#define NORTHSTAR_PREAMBLE_SIZE_SD		(16)
-#define NORTHSTAR_PREAMBLE_SIZE_DD		(32)
-#define NORTHSTAR_HEADER_SIZE_SD		(1)
-#define NORTHSTAR_HEADER_SIZE_DD		(2)
-#define NORTHSTAR_PAYLOAD_SIZE_SD		(256)
-#define NORTHSTAR_PAYLOAD_SIZE_DD		(512)
-#define NORTHSTAR_CHECKSUM_SIZE		(1)
-#define NORTHSTAR_ENCODED_SECTOR_SIZE_SD	(NORTHSTAR_HEADER_SIZE_SD + NORTHSTAR_PAYLOAD_SIZE_SD + NORTHSTAR_CHECKSUM_SIZE)
-#define NORTHSTAR_ENCODED_SECTOR_SIZE_DD	(NORTHSTAR_HEADER_SIZE_DD + NORTHSTAR_PAYLOAD_SIZE_DD + NORTHSTAR_CHECKSUM_SIZE)
+#define NORTHSTAR_PREAMBLE_SIZE_SD (16)
+#define NORTHSTAR_PREAMBLE_SIZE_DD (32)
+#define NORTHSTAR_HEADER_SIZE_SD (1)
+#define NORTHSTAR_HEADER_SIZE_DD (2)
+#define NORTHSTAR_PAYLOAD_SIZE_SD (256)
+#define NORTHSTAR_PAYLOAD_SIZE_DD (512)
+#define NORTHSTAR_CHECKSUM_SIZE (1)
+#define NORTHSTAR_ENCODED_SECTOR_SIZE_SD                    \
+    (NORTHSTAR_HEADER_SIZE_SD + NORTHSTAR_PAYLOAD_SIZE_SD + \
+        NORTHSTAR_CHECKSUM_SIZE)
+#define NORTHSTAR_ENCODED_SECTOR_SIZE_DD                    \
+    (NORTHSTAR_HEADER_SIZE_DD + NORTHSTAR_PAYLOAD_SIZE_DD + \
+        NORTHSTAR_CHECKSUM_SIZE)
 
 class Decoder;
 class Encoder;
@@ -29,7 +34,9 @@ class DecoderProto;
 
 extern uint8_t northstarChecksum(const Bytes& bytes);
 
-extern std::unique_ptr<Decoder> createNorthstarDecoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createNorthstarEncoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createNorthstarDecoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createNorthstarEncoder(
+    const EncoderProto& config);
 
 #endif /* NORTHSTAR */

arch/northstar/northstar.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message NorthstarDecoderProto {}
 

arch/rolandd20/decoder.cc

@@ -0,0 +1,52 @@
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
+#include "lib/core/bytes.h"
+#include "arch/rolandd20/rolandd20.h"
+#include <string.h>
+
+/* Sector header record:
+ *
+ * BF FF FF FF FF FF FE AB
+ *
+ * This encodes to:
+ *
+ *    e    d    5    5    5    5    5    5
+ * 1110 1101 0101 0101 0101 0101 0101 0101
+ *    5    5    5    5    5    5    5    5
+ * 0101 0101 0101 0101 0101 0101 0101 0101
+ *    5    5    5    5    5    5    5    5
+ * 0101 0101 0101 0101 0101 0101 0101 0101
+ *    5    5    5    4    4    4    4    5
+ * 0101 0101 0101 0100 0100 0100 0100 0101
+ */
+
+static const FluxPattern SECTOR_PATTERN(64, 0xed55555555555555LL);
+
+class RolandD20Decoder : public Decoder
+{
+public:
+    RolandD20Decoder(const DecoderProto& config): Decoder(config) {}
+
+    nanoseconds_t advanceToNextRecord() override
+    {
+        return seekToPattern(SECTOR_PATTERN);
+    }
+
+    void decodeSectorRecord() override
+    {
+        auto rawbits = readRawBits(256);
+        const auto& bytes = decodeFmMfm(rawbits);
+        fmt::print("{} ", _sector->clock);
+        hexdump(std::cout, bytes);
+    }
+};
+
+std::unique_ptr<Decoder> createRolandD20Decoder(const DecoderProto& config)
+{
+    return std::unique_ptr<Decoder>(new RolandD20Decoder(config));
+}

arch/rolandd20/rolandd20.h

@@ -0,0 +1,4 @@
+#pragma once
+
+extern std::unique_ptr<Decoder> createRolandD20Decoder(
+    const DecoderProto& config);

arch/rolandd20/rolandd20.proto

@@ -0,0 +1,5 @@
+syntax = "proto2";
+
+message RolandD20DecoderProto {}
+
+

arch/smaky6/decoder.cc

@@ -0,0 +1,155 @@
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "protocol.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/smaky6/smaky6.h"
+#include "lib/core/bytes.h"
+#include "lib/core/crc.h"
+#include "fmt/format.h"
+#include "lib/decoders/decoders.pb.h"
+#include <string.h>
+#include <algorithm>
+
+static const FluxPattern SECTOR_PATTERN(32, 0x54892aaa);
+
+class Smaky6Decoder : public Decoder
+{
+public:
+    Smaky6Decoder(const DecoderProto& config):
+        Decoder(config),
+        _config(config.smaky6())
+    {
+    }
+
+private:
+    /* Returns the sector ID of the _current_ sector. */
+    int advanceToNextSector()
+    {
+        auto previous = tell();
+        seekToIndexMark();
+        auto now = tell();
+        if ((now.ns() - previous.ns()) < 9e6)
+        {
+            seekToIndexMark();
+            auto next = tell();
+            if ((next.ns() - now.ns()) < 9e6)
+            {
+                /* We just found sector 0. */
+
+                _sectorId = 0;
+            }
+            else
+            {
+                /* Spurious... */
+
+                seek(now);
+            }
+        }
+
+        return _sectorId++;
+    }
+
+public:
+    void beginTrack() override
+    {
+        /* Find the start-of-track index marks, which will be an interval
+         * of about 6ms. */
+
+        seekToIndexMark();
+        _sectorId = 99;
+        for (;;)
+        {
+            auto pos = tell();
+            advanceToNextSector();
+            if (_sectorId < 99)
+            {
+                seek(pos);
+                break;
+            }
+
+            if (eof())
+                return;
+        }
+
+        /* Now we know where to start counting, start finding sectors. */
+
+        _sectorStarts.clear();
+        for (;;)
+        {
+            auto now = tell();
+            if (eof())
+                break;
+
+            int id = advanceToNextSector();
+            if (id < 16)
+                _sectorStarts.push_back(std::make_pair(id, now));
+        }
+
+        _sectorIndex = 0;
+    }
+
+    nanoseconds_t advanceToNextRecord() override
+    {
+        if (_sectorIndex == _sectorStarts.size())
+        {
+            seekToIndexMark();
+            return 0;
+        }
+
+        const auto& p = _sectorStarts[_sectorIndex++];
+        _sectorId = p.first;
+        seek(p.second);
+
+        nanoseconds_t clock = seekToPattern(SECTOR_PATTERN);
+        _sector->headerStartTime = tell().ns();
+
+        return clock;
+    }
+
+    void decodeSectorRecord() override
+    {
+        readRawBits(33);
+        const auto& rawbits = readRawBits(SMAKY6_RECORD_SIZE * 16);
+        if (rawbits.size() < SMAKY6_SECTOR_SIZE)
+            return;
+        const auto& rawbytes =
+            toBytes(rawbits).slice(0, SMAKY6_RECORD_SIZE * 16);
+
+        /* The Smaky bytes are stored backwards! Backwards! */
+
+        const auto& bytes =
+            decodeFmMfm(rawbits).slice(0, SMAKY6_RECORD_SIZE).reverseBits();
+        ByteReader br(bytes);
+
+        uint8_t track = br.read_8();
+        Bytes data = br.read(SMAKY6_SECTOR_SIZE);
+        uint8_t wantedChecksum = br.read_8();
+        uint8_t gotChecksum = sumBytes(data) & 0xff;
+
+        if (track != _sector->physicalTrack)
+            return;
+
+        _sector->logicalTrack = _sector->physicalTrack;
+        _sector->logicalSide = _sector->physicalSide;
+        _sector->logicalSector = _sectorId;
+
+        _sector->data = data;
+        _sector->status =
+            (wantedChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
+
+private:
+    const Smaky6DecoderProto& _config;
+    nanoseconds_t _startOfTrack;
+    std::vector<std::pair<int, Fluxmap::Position>> _sectorStarts;
+    int _sectorId;
+    int _sectorIndex;
+};
+
+std::unique_ptr<Decoder> createSmaky6Decoder(const DecoderProto& config)
+{
+    return std::unique_ptr<Decoder>(new Smaky6Decoder(config));
+}

arch/smaky6/smaky6.h

@@ -0,0 +1,9 @@
+#ifndef SMAKY6_H
+#define SMAKY6_H
+
+#define SMAKY6_SECTOR_SIZE 256
+#define SMAKY6_RECORD_SIZE (1 + SMAKY6_SECTOR_SIZE + 1)
+
+extern std::unique_ptr<Decoder> createSmaky6Decoder(const DecoderProto& config);
+
+#endif

arch/smaky6/smaky6.proto

@@ -0,0 +1,4 @@
+syntax = "proto2";
+
+message Smaky6DecoderProto {}
+

arch/tartu/decoder.cc

@@ -0,0 +1,81 @@
+#include "lib/core/globals.h"
+#include "lib/config/config.h"
+#include "lib/decoders/decoders.h"
+#include "arch/tartu/tartu.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
+#include <string.h>
+
+constexpr uint64_t HEADER_BITS = 0xaaaaaaaa44895554LL;
+constexpr uint64_t DATA_BITS = 0xaaaaaaaa44895545LL;
+
+static const FluxPattern HEADER_PATTERN(64, HEADER_BITS);
+static const FluxPattern DATA_PATTERN(64, DATA_BITS);
+
+const FluxMatchers ANY_RECORD_PATTERN{&HEADER_PATTERN, &DATA_PATTERN};
+
+class TartuDecoder : public Decoder
+{
+public:
+    TartuDecoder(const DecoderProto& config):
+        Decoder(config),
+        _config(config.tartu())
+    {
+    }
+
+    void beginTrack() override {}
+
+    nanoseconds_t advanceToNextRecord() override
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
+
+    void decodeSectorRecord() override
+    {
+        if (readRaw64() != HEADER_BITS)
+            return;
+
+        auto bits = readRawBits(16 * 4);
+        auto bytes = decodeFmMfm(bits).slice(0, 4);
+
+        ByteReader br(bytes);
+        uint8_t track = br.read_8();
+        _sector->logicalTrack = track >> 1;
+        _sector->logicalSide = track & 1;
+        br.skip(1); /* seems always to be 1 */
+        _sector->logicalSector = br.read_8();
+        uint8_t wantChecksum = br.read_8();
+        uint8_t gotChecksum = ~sumBytes(bytes.slice(0, 3));
+
+        if (wantChecksum == gotChecksum)
+            _sector->status = Sector::DATA_MISSING;
+
+        _sector->status = Sector::DATA_MISSING;
+    }
+
+    void decodeDataRecord() override
+    {
+        if (readRaw64() != DATA_BITS)
+            return;
+
+        const auto& bits = readRawBits(129 * 16);
+        const auto& bytes = decodeFmMfm(bits).slice(0, 129);
+        _sector->data = bytes.slice(0, 128);
+
+        uint8_t wantChecksum = bytes.reader().seek(128).read_8();
+        uint8_t gotChecksum = ~sumBytes(_sector->data);
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
+
+private:
+    const TartuDecoderProto& _config;
+};
+
+std::unique_ptr<Decoder> createTartuDecoder(const DecoderProto& config)
+{
+    return std::unique_ptr<Decoder>(new TartuDecoder(config));
+}

arch/tartu/encoder.cc

@@ -0,0 +1,115 @@
+#include "lib/core/globals.h"
+#include "lib/config/config.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/tartu/tartu.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/sector.h"
+#include <string.h>
+
+class TartuEncoder : public Encoder
+{
+public:
+    TartuEncoder(const EncoderProto& config):
+        Encoder(config),
+        _config(config.tartu())
+    {
+    }
+
+    std::unique_ptr<Fluxmap> encode(std::shared_ptr<const TrackInfo>& trackInfo,
+        const std::vector<std::shared_ptr<const Sector>>& sectors,
+        const Image& image) override
+    {
+        _clockRateUs = _config.clock_period_us();
+        int bitsPerRevolution =
+            (_config.target_rotational_period_ms() * 1000.0) / _clockRateUs;
+
+        const auto& sector = *sectors.begin();
+        _bits.resize(bitsPerRevolution);
+        _cursor = 0;
+
+        writeFillerRawBitsUs(_config.gap1_us());
+        bool first = true;
+        for (const auto& sectorData : sectors)
+        {
+            if (!first)
+                writeFillerRawBitsUs(_config.gap4_us());
+            first = false;
+            writeSector(sectorData);
+        }
+
+        if (_cursor > _bits.size())
+            error("track data overrun");
+        writeFillerRawBitsUs(_config.target_rotational_period_ms() * 1000.0);
+
+        std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+        fluxmap->appendBits(_bits,
+            calculatePhysicalClockPeriod(_clockRateUs * 1e3,
+                _config.target_rotational_period_ms() * 1e6));
+        return fluxmap;
+    }
+
+private:
+    void writeBytes(const Bytes& bytes)
+    {
+        encodeMfm(_bits, _cursor, bytes, _lastBit);
+    }
+
+    void writeRawBits(uint64_t data, int width)
+    {
+        _cursor += width;
+        _lastBit = data & 1;
+        for (int i = 0; i < width; i++)
+        {
+            unsigned pos = _cursor - i - 1;
+            if (pos < _bits.size())
+                _bits[pos] = data & 1;
+            data >>= 1;
+        }
+    }
+
+    void writeFillerRawBitsUs(double us)
+    {
+        unsigned count = (us / _clockRateUs) / 2;
+        for (int i = 0; i < count; i++)
+            writeRawBits(0b10, 2);
+    };
+
+    void writeSector(const std::shared_ptr<const Sector>& sectorData)
+    {
+        writeRawBits(_config.header_marker(), 64);
+        {
+            Bytes bytes;
+            ByteWriter bw(bytes);
+            bw.write_8(
+                (sectorData->logicalTrack << 1) | sectorData->logicalSide);
+            bw.write_8(1);
+            bw.write_8(sectorData->logicalSector);
+            bw.write_8(~sumBytes(bytes.slice(0, 3)));
+            writeBytes(bytes);
+        }
+
+        writeFillerRawBitsUs(_config.gap3_us());
+        writeRawBits(_config.data_marker(), 64);
+        {
+            Bytes bytes;
+            ByteWriter bw(bytes);
+            bw.append(sectorData->data);
+            bw.write_8(~sumBytes(bytes.slice(0, sectorData->data.size())));
+            writeBytes(bytes);
+        }
+    }
+
+private:
+    const TartuEncoderProto& _config;
+    double _clockRateUs;
+    std::vector<bool> _bits;
+    unsigned _cursor;
+    bool _lastBit;
+};
+
+std::unique_ptr<Encoder> createTartuEncoder(const EncoderProto& config)
+{
+    return std::unique_ptr<Encoder>(new TartuEncoder(config));
+}

arch/tartu/tartu.h

@@ -0,0 +1,7 @@
+#ifndef TARTU_H
+#define TARTU_H
+
+extern std::unique_ptr<Decoder> createTartuDecoder(const DecoderProto& config);
+extern std::unique_ptr<Encoder> createTartuEncoder(const EncoderProto& config);
+
+#endif

arch/tartu/tartu.proto

@@ -0,0 +1,27 @@
+syntax = "proto2";
+
+import "lib/config/common.proto";
+
+message TartuDecoderProto {}
+
+message TartuEncoderProto {
+    optional double clock_period_us = 1
+        [ default = 2.0, (help) = "clock rate on the real device (for MFM)" ];
+    optional double target_rotational_period_ms = 2
+        [ default=200, (help) = "rotational period of target disk" ];
+    optional double gap1_us = 3
+        [ default = 1200,
+          (help) = "size of gap 1 (the post-index gap)" ];
+    optional double gap3_us = 4
+        [ default = 150,
+          (help) = "size of gap 3 (the pre-data gap)" ];
+    optional double gap4_us = 5
+        [ default = 180,
+          (help) = "size of gap 4 (the post-data or format gap)" ];
+    optional uint64 header_marker = 6
+        [ default = 0xaaaaaaaa44895554,
+          (help) = "64-bit raw bit pattern of header record marker" ];
+    optional uint64 data_marker = 7
+        [ default = 0xaaaaaaaa44895545,
+          (help) = "64-bit raw bit pattern of data record marker" ];
+}

arch/tids990/decoder.cc

@@ -1,13 +1,14 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "tids990/tids990.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
-#include "sector.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/tids990/tids990.h"
+#include "lib/core/crc.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
+#include "lib/data/sector.h"
 #include <string.h>
-#include <fmt/format.h>
+#include "fmt/format.h"
 
 /* The Texas Instruments DS990 uses MFM with a scheme similar to a simplified
  * version of the IBM record scheme (it's actually easier to parse than IBM).
@@ -38,61 +39,63 @@ const FluxPattern SECTOR_RECORD_PATTERN(32, 0x11112244);
 const uint16_t DATA_ID = 0x550b;
 const FluxPattern DATA_RECORD_PATTERN(32, 0x11112245);
 
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 class Tids990Decoder : public Decoder
 {
 public:
-	Tids990Decoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    Tids990Decoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		auto bits = readRawBits(TIDS990_SECTOR_RECORD_SIZE*16);
-		auto bytes = decodeFmMfm(bits).slice(0, TIDS990_SECTOR_RECORD_SIZE);
+    {
+        auto bits = readRawBits(TIDS990_SECTOR_RECORD_SIZE * 16);
+        auto bytes = decodeFmMfm(bits).slice(0, TIDS990_SECTOR_RECORD_SIZE);
 
-		ByteReader br(bytes);
-		if (br.read_be16() != SECTOR_ID)
-			return;
+        ByteReader br(bytes);
+        if (br.read_be16() != SECTOR_ID)
+            return;
 
-		uint16_t gotChecksum = crc16(CCITT_POLY, bytes.slice(1, TIDS990_SECTOR_RECORD_SIZE-3));
+        uint16_t gotChecksum =
+            crc16(CCITT_POLY, bytes.slice(1, TIDS990_SECTOR_RECORD_SIZE - 3));
 
-		_sector->logicalSide = br.read_8() >> 3;
-		_sector->logicalTrack = br.read_8();
-		br.read_8(); /* number of sectors per track */
-		_sector->logicalSector = br.read_8();
-		br.read_be16(); /* sector size */
-		uint16_t wantChecksum = br.read_be16();
+        _sector->logicalSide = br.read_8() >> 3;
+        _sector->logicalTrack = br.read_8();
+        br.read_8(); /* number of sectors per track */
+        _sector->logicalSector = br.read_8();
+        br.read_be16(); /* sector size */
+        uint16_t wantChecksum = br.read_be16();
 
-		if (wantChecksum == gotChecksum)
-			_sector->status = Sector::DATA_MISSING; /* correct but unintuitive */
-	}
+        if (wantChecksum == gotChecksum)
+            _sector->status =
+                Sector::DATA_MISSING; /* correct but unintuitive */
+    }
 
-	void decodeDataRecord() override
-	{
-		auto bits = readRawBits(TIDS990_DATA_RECORD_SIZE*16);
-		auto bytes = decodeFmMfm(bits).slice(0, TIDS990_DATA_RECORD_SIZE);
+    void decodeDataRecord() override
+    {
+        auto bits = readRawBits(TIDS990_DATA_RECORD_SIZE * 16);
+        auto bytes = decodeFmMfm(bits).slice(0, TIDS990_DATA_RECORD_SIZE);
 
-		ByteReader br(bytes);
-		if (br.read_be16() != DATA_ID)
-			return;
+        ByteReader br(bytes);
+        if (br.read_be16() != DATA_ID)
+            return;
 
-		uint16_t gotChecksum = crc16(CCITT_POLY, bytes.slice(1, TIDS990_DATA_RECORD_SIZE-3));
+        uint16_t gotChecksum =
+            crc16(CCITT_POLY, bytes.slice(1, TIDS990_DATA_RECORD_SIZE - 3));
 
-		_sector->data = br.read(TIDS990_PAYLOAD_SIZE);
-		uint16_t wantChecksum = br.read_be16();
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = br.read(TIDS990_PAYLOAD_SIZE);
+        uint16_t wantChecksum = br.read_be16();
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createTids990Decoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new Tids990Decoder(config));
+    return std::unique_ptr<Decoder>(new Tids990Decoder(config));
 }
-

arch/tids990/encoder.cc

@@ -1,13 +1,12 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "tids990.h"
-#include "crc.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/tids990/tids990.h"
+#include "lib/core/crc.h"
+#include "lib/data/image.h"
 #include "arch/tids990/tids990.pb.h"
 #include "lib/encoders/encoders.pb.h"
-#include <fmt/format.h>
+#include "fmt/format.h"
 
 static int charToInt(char c)
 {
@@ -127,14 +126,14 @@ public:
         }
 
         if (_cursor >= _bits.size())
-            Error() << "track data overrun";
+            error("track data overrun");
         while (_cursor < _bits.size())
             writeBytes(1, 0x55);
 
         auto fluxmap = std::make_unique<Fluxmap>();
         fluxmap->appendBits(_bits,
-            calculatePhysicalClockPeriod(clockRateUs * 1e3,
-                _config.rotational_period_ms() * 1e6));
+            calculatePhysicalClockPeriod(
+                clockRateUs * 1e3, _config.rotational_period_ms() * 1e6));
         return fluxmap;
     }
 
@@ -145,8 +144,7 @@ private:
     bool _lastBit;
 };
 
-std::unique_ptr<Encoder> createTids990Encoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createTids990Encoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new Tids990Encoder(config));
 }

arch/tids990/tids990.h

@@ -1,18 +1,18 @@
 #ifndef TIDS990_H
 #define TIDS990_H
 
-#define TIDS990_PAYLOAD_SIZE       288 /* bytes */
-#define TIDS990_SECTOR_RECORD_SIZE 10 /* bytes */
-#define TIDS990_DATA_RECORD_SIZE   (TIDS990_PAYLOAD_SIZE + 4) /* bytes */
+#define TIDS990_PAYLOAD_SIZE 288                            /* bytes */
+#define TIDS990_SECTOR_RECORD_SIZE 10                       /* bytes */
+#define TIDS990_DATA_RECORD_SIZE (TIDS990_PAYLOAD_SIZE + 4) /* bytes */
 
 class Encoder;
 class Decoder;
 class DecoderProto;
 class EncoderProto;
 
-extern std::unique_ptr<Decoder> createTids990Decoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createTids990Encoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createTids990Decoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createTids990Encoder(
+    const EncoderProto& config);
 
 #endif
-
-

arch/tids990/tids990.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message Tids990DecoderProto {}
 

arch/victor9k/decoder.cc

@@ -1,28 +1,31 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "victor9k.h"
-#include "crc.h"
-#include "bytes.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/victor9k/victor9k.h"
+#include "lib/core/crc.h"
+#include "lib/core/bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
 const FluxPattern SECTOR_RECORD_PATTERN(32, VICTOR9K_SECTOR_RECORD);
 const FluxPattern DATA_RECORD_PATTERN(32, VICTOR9K_DATA_RECORD);
-const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+const FluxMatchers ANY_RECORD_PATTERN(
+    {&SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN});
 
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
     {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
+#define GCR_ENTRY(gcr, data) \
+    case gcr:                \
+        return data;
+#include "data_gcr.h"
+#undef GCR_ENTRY
     }
     return -1;
 }
@@ -37,11 +40,11 @@ static Bytes decode(const std::vector<bool>& bits)
     while (ii != bits.end())
     {
         uint8_t inputfifo = 0;
-        for (size_t i=0; i<5; i++)
+        for (size_t i = 0; i < 5; i++)
         {
             if (ii == bits.end())
                 break;
-            inputfifo = (inputfifo<<1) | *ii++;
+            inputfifo = (inputfifo << 1) | *ii++;
         }
 
         uint8_t decoded = decode_data_gcr(inputfifo);
@@ -55,63 +58,62 @@ static Bytes decode(const std::vector<bool>& bits)
 class Victor9kDecoder : public Decoder
 {
 public:
-	Victor9kDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    Victor9kDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		return seekToPattern(ANY_RECORD_PATTERN);
-	}
+    {
+        return seekToPattern(ANY_RECORD_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		/* Check the ID. */
+    {
+        /* Check the ID. */
 
-		if (readRaw32() != VICTOR9K_SECTOR_RECORD)
-			return;
+        if (readRaw32() != VICTOR9K_SECTOR_RECORD)
+            return;
 
-		/* Read header. */
+        /* Read header. */
 
-		auto bytes = decode(readRawBits(3*10)).slice(0, 3);
+        auto bytes = decode(readRawBits(3 * 10)).slice(0, 3);
 
-		uint8_t rawTrack = bytes[0];
-		_sector->logicalSector = bytes[1];
-		uint8_t gotChecksum = bytes[2];
+        uint8_t rawTrack = bytes[0];
+        _sector->logicalSector = bytes[1];
+        uint8_t gotChecksum = bytes[2];
 
-		_sector->logicalTrack = rawTrack & 0x7f;
-		_sector->logicalSide = rawTrack >> 7;
-		uint8_t wantChecksum = bytes[0] + bytes[1];
-		if ((_sector->logicalSector > 20) || (_sector->logicalTrack > 85) || (_sector->logicalSide > 1))
-			return;
-					
-		if (wantChecksum == gotChecksum)
-			_sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
-	}
+        _sector->logicalTrack = rawTrack & 0x7f;
+        _sector->logicalSide = rawTrack >> 7;
+        uint8_t wantChecksum = bytes[0] + bytes[1];
+        if ((_sector->logicalSector > 20) || (_sector->logicalTrack > 85) ||
+            (_sector->logicalSide > 1))
+            return;
+
+        if (wantChecksum == gotChecksum)
+            _sector->status =
+                Sector::DATA_MISSING; /* unintuitive but correct */
+    }
 
     void decodeDataRecord() override
-	{
-		/* Check the ID. */
+    {
+        /* Check the ID. */
 
-		if (readRaw32() != VICTOR9K_DATA_RECORD)
-			return;
+        if (readRaw32() != VICTOR9K_DATA_RECORD)
+            return;
 
-		/* Read data. */
+        /* Read data. */
 
-		auto bytes = decode(readRawBits((VICTOR9K_SECTOR_LENGTH+4)*10))
-			.slice(0, VICTOR9K_SECTOR_LENGTH+4);
-		ByteReader br(bytes);
+        auto bytes = decode(readRawBits((VICTOR9K_SECTOR_LENGTH + 4) * 10))
+                         .slice(0, VICTOR9K_SECTOR_LENGTH + 4);
+        ByteReader br(bytes);
 
-		_sector->data = br.read(VICTOR9K_SECTOR_LENGTH);
-		uint16_t gotChecksum = sumBytes(_sector->data);
-		uint16_t wantChecksum = br.read_le16();
-		_sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->data = br.read(VICTOR9K_SECTOR_LENGTH);
+        uint16_t gotChecksum = sumBytes(_sector->data);
+        uint16_t wantChecksum = br.read_le16();
+        _sector->status =
+            (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createVictor9kDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new Victor9kDecoder(config));
+    return std::unique_ptr<Decoder>(new Victor9kDecoder(config));
 }
-
-

arch/victor9k/encoder.cc

@@ -1,17 +1,17 @@
-#include "globals.h"
-#include "decoders/decoders.h"
-#include "encoders/encoders.h"
-#include "victor9k.h"
-#include "crc.h"
-#include "sector.h"
-#include "readerwriter.h"
-#include "image.h"
+#include "lib/core/globals.h"
+#include "lib/core/utils.h"
+#include "lib/decoders/decoders.h"
+#include "lib/encoders/encoders.h"
+#include "arch/victor9k/victor9k.h"
+#include "lib/core/crc.h"
+#include "lib/data/sector.h"
+#include "lib/data/image.h"
 #include "fmt/format.h"
 #include "arch/victor9k/victor9k.pb.h"
 #include "lib/encoders/encoders.pb.h"
-#include "lib/layout.h"
+#include "lib/data/layout.h"
 #include <ctype.h>
-#include "bytes.h"
+#include "lib/core/bytes.h"
 
 static bool lastBit;
 
@@ -169,14 +169,15 @@ public:
         const Image& image) override
     {
         Victor9kEncoderProto::TrackdataProto trackdata;
-        getTrackFormat(trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
+        getTrackFormat(
+            trackdata, trackInfo->logicalTrack, trackInfo->logicalSide);
 
         unsigned bitsPerRevolution = (trackdata.rotational_period_ms() * 1e3) /
                                      trackdata.clock_period_us();
         std::vector<bool> bits(bitsPerRevolution);
-        nanoseconds_t clockPeriod = calculatePhysicalClockPeriod(
-            trackdata.clock_period_us() * 1e3,
-            trackdata.rotational_period_ms() * 1e6);
+        nanoseconds_t clockPeriod =
+            calculatePhysicalClockPeriod(trackdata.clock_period_us() * 1e3,
+                trackdata.rotational_period_ms() * 1e6);
         unsigned cursor = 0;
 
         fillBitmapTo(bits,
@@ -189,8 +190,7 @@ public:
             write_sector(bits, cursor, trackdata, *sector);
 
         if (cursor >= bits.size())
-            Error() << fmt::format(
-                "track data overrun by {} bits", cursor - bits.size());
+            error("track data overrun by {} bits", cursor - bits.size());
         fillBitmapTo(bits, cursor, bits.size(), {true, false});
 
         std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
@@ -202,8 +202,7 @@ private:
     const Victor9kEncoderProto& _config;
 };
 
-std::unique_ptr<Encoder> createVictor9kEncoder(
-    const EncoderProto& config)
+std::unique_ptr<Encoder> createVictor9kEncoder(const EncoderProto& config)
 {
     return std::unique_ptr<Encoder>(new Victor9kEncoder(config));
 }

arch/victor9k/victor9k.h

@@ -13,12 +13,14 @@ class DecoderProto;
 
 /* ... 1101 0100 1001
  *       ^^ ^^^^ ^^^^ ten bit IO byte */
-#define VICTOR9K_DATA_RECORD   0xfffffd49
+#define VICTOR9K_DATA_RECORD 0xfffffd49
 #define VICTOR9K_DATA_ID 0x8
 
 #define VICTOR9K_SECTOR_LENGTH 512
 
-extern std::unique_ptr<Decoder> createVictor9kDecoder(const DecoderProto& config);
-extern std::unique_ptr<Encoder> createVictor9kEncoder(const EncoderProto& config);
+extern std::unique_ptr<Decoder> createVictor9kDecoder(
+    const DecoderProto& config);
+extern std::unique_ptr<Encoder> createVictor9kEncoder(
+    const EncoderProto& config);
 
 #endif

arch/victor9k/victor9k.proto

@@ -1,6 +1,6 @@
 syntax = "proto2";
 
-import "lib/common.proto";
+import "lib/config/common.proto";
 
 message Victor9kDecoderProto {}
 

arch/zilogmcz/decoder.cc

@@ -1,12 +1,13 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "decoders/fluxmapreader.h"
+#include "lib/core/globals.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/fluxmapreader.h"
+#include "lib/data/fluxpattern.h"
 #include "protocol.h"
-#include "decoders/decoders.h"
-#include "sector.h"
-#include "zilogmcz.h"
-#include "bytes.h"
-#include "crc.h"
+#include "lib/decoders/decoders.h"
+#include "lib/data/sector.h"
+#include "arch/zilogmcz/zilogmcz.h"
+#include "lib/core/bytes.h"
+#include "lib/core/crc.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
@@ -16,42 +17,40 @@ static const FluxPattern SECTOR_START_PATTERN(16, 0xaaab);
 class ZilogMczDecoder : public Decoder
 {
 public:
-	ZilogMczDecoder(const DecoderProto& config):
-		Decoder(config)
-	{}
+    ZilogMczDecoder(const DecoderProto& config): Decoder(config) {}
 
     nanoseconds_t advanceToNextRecord() override
-	{
-		seekToIndexMark();
-		return seekToPattern(SECTOR_START_PATTERN);
-	}
+    {
+        seekToIndexMark();
+        return seekToPattern(SECTOR_START_PATTERN);
+    }
 
     void decodeSectorRecord() override
-	{
-		readRawBits(14);
+    {
+        readRawBits(14);
 
-		auto rawbits = readRawBits(140*16);
-		auto bytes = decodeFmMfm(rawbits).slice(0, 140);
-		ByteReader br(bytes);
+        auto rawbits = readRawBits(140 * 16);
+        auto bytes = decodeFmMfm(rawbits).slice(0, 140);
+        ByteReader br(bytes);
 
-		_sector->logicalSector = br.read_8() & 0x1f;
-		_sector->logicalSide = 0;
-		_sector->logicalTrack = br.read_8() & 0x7f;
-		if (_sector->logicalSector > 31)
-			return;
-		if (_sector->logicalTrack > 80)
-			return;
+        _sector->logicalSector = br.read_8() & 0x1f;
+        _sector->logicalSide = 0;
+        _sector->logicalTrack = br.read_8() & 0x7f;
+        if (_sector->logicalSector > 31)
+            return;
+        if (_sector->logicalTrack > 80)
+            return;
 
-		_sector->data = br.read(132);
-		uint16_t wantChecksum = br.read_be16();
-		uint16_t gotChecksum = crc16(MODBUS_POLY, 0x0000, bytes.slice(0, 134));
+        _sector->data = br.read(132);
+        uint16_t wantChecksum = br.read_be16();
+        uint16_t gotChecksum = crc16(MODBUS_POLY, 0x0000, bytes.slice(0, 134));
 
-		_sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-	}
+        _sector->status =
+            (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    }
 };
 
 std::unique_ptr<Decoder> createZilogMczDecoder(const DecoderProto& config)
 {
-	return std::unique_ptr<Decoder>(new ZilogMczDecoder(config));
+    return std::unique_ptr<Decoder>(new ZilogMczDecoder(config));
 }
-

arch/zilogmcz/zilogmcz.h

@@ -1,8 +1,7 @@
 #ifndef ZILOGMCZ_H
 #define ZILOGMCZ_H
 
-extern std::unique_ptr<Decoder> createZilogMczDecoder(const DecoderProto& config);
+extern std::unique_ptr<Decoder> createZilogMczDecoder(
+    const DecoderProto& config);
 
 #endif
-
-

build.lua

@@ -1,192 +0,0 @@
-vars.cflags = { "$(CFLAGS)" }
-vars.cxxflags = { "$(CXXFLAGS)" }
-vars.ldflags = { "-pthread" }
-
-include "build/protobuf.lua"
-include "build/dependency.lua"
-include "build/tests.lua"
-
-dependency {
-	name = "fmt_dep",
-	pkg_config = "fmt",
-}
-
-dependency {
-	name = "stb_dep",
-	pkg_config = "stb",
-	fallback = "dep/stb+stb"
-}
-
-dependency {
-	name = "protobuf_dep",
-	pkg_config = "protobuf"
-}
-
-dependency {
-	name = "zlib_dep",
-	pkg_config = "zlib"
-}
-
-proto_cc_library {
-	name = "config_lib",
-	srcs = {
-		"./lib/common.proto",
-		"./lib/config.proto",
-		"./lib/decoders/decoders.proto",
-		"./lib/drive.proto",
-		"./lib/encoders/encoders.proto",
-		"./lib/fl2.proto",
-		"./lib/fluxsink/fluxsink.proto",
-		"./lib/fluxsource/fluxsource.proto",
-		"./lib/imagereader/imagereader.proto",
-		"./lib/imagewriter/imagewriter.proto",
-		"./lib/mapper.proto",
-		"./lib/usb/usb.proto",
-		"./arch/aeslanier/aeslanier.proto",
-		"./arch/agat/agat.proto",
-		"./arch/amiga/amiga.proto",
-		"./arch/apple2/apple2.proto",
-		"./arch/brother/brother.proto",
-		"./arch/c64/c64.proto",
-		"./arch/f85/f85.proto",
-		"./arch/fb100/fb100.proto",
-		"./arch/ibm/ibm.proto",
-		"./arch/macintosh/macintosh.proto",
-		"./arch/micropolis/micropolis.proto",
-		"./arch/mx/mx.proto",
-		"./arch/northstar/northstar.proto",
-		"./arch/tids990/tids990.proto",
-		"./arch/victor9k/victor9k.proto",
-		"./arch/zilogmcz/zilogmcz.proto",
-	}
-}
-
-clibrary {
-	name = "protocol_lib",
-	hdrs = { "./protocol.h" }
-}
-
-clibrary {
-	name = "libfluxengine",
-	srcs = {
-		"./arch/aeslanier/decoder.cc",
-		"./arch/agat/agat.cc",
-		"./arch/agat/decoder.cc",
-		"./arch/amiga/amiga.cc",
-		"./arch/amiga/decoder.cc",
-		"./arch/amiga/encoder.cc",
-		"./arch/apple2/decoder.cc",
-		"./arch/apple2/encoder.cc",
-		"./arch/brother/decoder.cc",
-		"./arch/brother/encoder.cc",
-		"./arch/c64/c64.cc",
-		"./arch/c64/decoder.cc",
-		"./arch/c64/encoder.cc",
-		"./arch/f85/decoder.cc",
-		"./arch/fb100/decoder.cc",
-		"./arch/ibm/decoder.cc",
-		"./arch/ibm/encoder.cc",
-		"./arch/macintosh/decoder.cc",
-		"./arch/macintosh/encoder.cc",
-		"./arch/micropolis/decoder.cc",
-		"./arch/micropolis/encoder.cc",
-		"./arch/mx/decoder.cc",
-		"./arch/northstar/decoder.cc",
-		"./arch/northstar/encoder.cc",
-		"./arch/tids990/decoder.cc",
-		"./arch/tids990/encoder.cc",
-		"./arch/victor9k/decoder.cc",
-		"./arch/victor9k/encoder.cc",
-		"./arch/zilogmcz/decoder.cc",
-		"./lib/bitmap.cc",
-		"./lib/bytes.cc",
-		"./lib/crc.cc",
-		"./lib/csvreader.cc",
-		"./lib/decoders/decoders.cc",
-		"./lib/decoders/fluxdecoder.cc",
-		"./lib/decoders/fluxmapreader.cc",
-		"./lib/decoders/fmmfm.cc",
-		"./lib/encoders/encoders.cc",
-		"./lib/flags.cc",
-		"./lib/fluxmap.cc",
-		"./lib/fluxsink/aufluxsink.cc",
-		"./lib/fluxsink/fl2fluxsink.cc",
-		"./lib/fluxsink/fluxsink.cc",
-		"./lib/fluxsink/hardwarefluxsink.cc",
-		"./lib/fluxsink/scpfluxsink.cc",
-		"./lib/fluxsink/vcdfluxsink.cc",
-		"./lib/fluxsource/cwffluxsource.cc",
-		"./lib/fluxsource/erasefluxsource.cc",
-		"./lib/fluxsource/fl2fluxsource.cc",
-		"./lib/fluxsource/fluxsource.cc",
-		"./lib/fluxsource/hardwarefluxsource.cc",
-		"./lib/fluxsource/kryoflux.cc",
-		"./lib/fluxsource/kryofluxfluxsource.cc",
-		"./lib/fluxsource/scpfluxsource.cc",
-		"./lib/fluxsource/testpatternfluxsource.cc",
-		"./lib/globals.cc",
-		"./lib/hexdump.cc",
-		"./lib/image.cc",
-		"./lib/imagereader/d64imagereader.cc",
-		"./lib/imagereader/d88imagereader.cc",
-		"./lib/imagereader/dimimagereader.cc",
-		"./lib/imagereader/diskcopyimagereader.cc",
-		"./lib/imagereader/fdiimagereader.cc",
-		"./lib/imagereader/imagereader.cc",
-		"./lib/imagereader/imdimagereader.cc",
-		"./lib/imagereader/imgimagereader.cc",
-		"./lib/imagereader/jv3imagereader.cc",
-		"./lib/imagereader/nfdimagereader.cc",
-		"./lib/imagereader/nsiimagereader.cc",
-		"./lib/imagereader/td0imagereader.cc",
-		"./lib/imagewriter/d64imagewriter.cc",
-		"./lib/imagewriter/d88imagewriter.cc",
-		"./lib/imagewriter/diskcopyimagewriter.cc",
-		"./lib/imagewriter/imagewriter.cc",
-		"./lib/imagewriter/imgimagewriter.cc",
-		"./lib/imagewriter/ldbsimagewriter.cc",
-		"./lib/imagewriter/nsiimagewriter.cc",
-		"./lib/imagewriter/rawimagewriter.cc",
-		"./lib/imginputoutpututils.cc",
-		"./lib/ldbs.cc",
-		"./lib/logger.cc",
-		"./lib/mapper.cc",
-		"./lib/proto.cc",
-		"./lib/readerwriter.cc",
-		"./lib/sector.cc",
-		"./lib/usb/fluxengineusb.cc",
-		"./lib/usb/greaseweazle.cc",
-		"./lib/usb/greaseweazleusb.cc",
-		"./lib/usb/serial.cc",
-		"./lib/usb/usb.cc",
-		"./lib/usb/usbfinder.cc",
-		"./lib/utils.cc",
-		"protocol.h",
-	},
-	deps = {
-		"+config_lib",
-		"+protocol_lib",
-		"+fmt_dep",
-		"+protobuf_dep",
-		"+zlib_dep",
-		"dep/libusbp+libusbp",
-	},
-	dep_cflags = { "-Ilib", "-Iarch", "-I." },
-	vars = {
-		["+cflags"] = { "-Ilib", "-Iarch", "-I." }
-	}
-}
-
-installable {
-	name = "all",
-	map = {
-		["fluxengine"] = "src+fluxengine",
-		["fluxengine-gui"] = "src/gui+fluxengine",
-		["brother120tool"] = "tools+brother120tool",
-		["brother240tool"] = "tools+brother240tool",
-		["upgrade-flux-file"] = "tools+upgrade-flux-file",
-	}
-}
-
-include "tests/build.lua"
-

build.py

@@ -0,0 +1,121 @@
+from build.ab import export
+from build.c import clibrary, cxxlibrary
+from build.protobuf import proto, protocc
+from build.pkg import package
+from build.utils import test
+from glob import glob
+import config
+import re
+
+# Hack for building on Fedora/WSL; executables get the .exe extension,
+# but the build system detects it as Linux.
+import build.toolchain
+
+toolchain.Toolchain.EXE = "$(EXT)"
+
+package(name="protobuf_lib", package="protobuf")
+package(name="z_lib", package="zlib")
+package(name="fmt_lib", package="fmt", fallback="dep/fmt")
+package(name="sqlite3_lib", package="sqlite3")
+
+clibrary(name="protocol", hdrs={"protocol.h": "./protocol.h"})
+
+corpustests = []
+if not glob("../fluxengine-testdata/data"):
+    print("fluxengine-testdata not found; skipping corpus tests")
+else:
+    corpus = [
+        ("acorndfs", "", "--200"),
+        ("agat", "", ""),
+        ("amiga", "", ""),
+        ("apple2", "", "--140 --drivetype=40"),
+        ("atarist", "", "--360"),
+        ("atarist", "", "--370"),
+        ("atarist", "", "--400"),
+        ("atarist", "", "--410"),
+        ("atarist", "", "--720"),
+        ("atarist", "", "--740"),
+        ("atarist", "", "--800"),
+        ("atarist", "", "--820"),
+        ("bk", "", ""),
+        ("brother", "", "--120 --drivetype=40"),
+        ("brother", "", "--240"),
+        (
+            "commodore",
+            "scripts/commodore1541_test.textpb",
+            "--171 --drivetype=40",
+        ),
+        (
+            "commodore",
+            "scripts/commodore1541_test.textpb",
+            "--192 --drivetype=40",
+        ),
+        ("commodore", "", "--800"),
+        ("commodore", "", "--1620"),
+        ("hplif", "", "--264"),
+        ("hplif", "", "--608"),
+        ("hplif", "", "--616"),
+        ("hplif", "", "--770"),
+        ("ibm", "", "--1200"),
+        ("ibm", "", "--1232"),
+        ("ibm", "", "--1440"),
+        ("ibm", "", "--1680"),
+        ("ibm", "", "--180 --drivetype=40"),
+        ("ibm", "", "--160 --drivetype=40"),
+        ("ibm", "", "--320 --drivetype=40"),
+        ("ibm", "", "--360 --drivetype=40"),
+        ("ibm", "", "--720_96"),
+        ("ibm", "", "--720_135"),
+        ("mac", "scripts/mac400_test.textpb", "--400"),
+        ("mac", "scripts/mac800_test.textpb", "--800"),
+        ("n88basic", "", ""),
+        ("rx50", "", ""),
+        ("tartu", "", "--390 --drivetype=40"),
+        ("tartu", "", "--780"),
+        ("tids990", "", ""),
+        ("victor9k", "", "--612"),
+        ("victor9k", "", "--1224"),
+    ]
+
+    for c in corpus:
+        name = re.sub(r"[^a-zA-Z0-9]", "_", "".join(c), 0)
+        corpustests += [
+            test(
+                name=f"corpustest_{name}_{format}",
+                ins=["src+fluxengine"],
+                deps=["scripts/encodedecodetest.sh"],
+                commands=[
+                    "$[deps[0]] "
+                    + c[0]
+                    + " "
+                    + format
+                    + " $[ins[0]] '"
+                    + c[1]
+                    + "' '"
+                    + c[2]
+                    + "' $[dirname(filenameof(outs[0]))] > /dev/null"
+                ],
+                label="CORPUSTEST",
+            )
+            for format in ["scp", "flux"]
+        ]
+
+export(
+    name="all",
+    items={
+        "fluxengine$(EXT)": "src+fluxengine",
+        "fluxengine-gui$(EXT)": "src/gui",
+        "brother120tool$(EXT)": "tools+brother120tool",
+        "brother240tool$(EXT)": "tools+brother240tool",
+        "upgrade-flux-file$(EXT)": "tools+upgrade-flux-file",
+    }
+    | (
+        {
+            "FluxEngine.pkg": "src/gui+fluxengine_pkg",
+            "FluxEngine.app.zip": "src/gui+fluxengine_app_zip",
+        }
+        if config.osx
+        else {}
+    ),
+    deps=["tests", "src/formats+docs", "scripts+mkdocindex"] + corpustests,
+)

build/_objectify.py

@@ -0,0 +1,19 @@
+import sys
+from functools import partial
+
+if len(sys.argv) != 3:
+    sys.exit("Usage: %s <file> <symbol>" % sys.argv[0])
+filename = sys.argv[1]
+symbol = sys.argv[2]
+
+print("const uint8_t " + symbol + "[] = {")
+n = 0
+with open(filename, "rb") as in_file:
+    for c in iter(partial(in_file.read, 1), b""):
+        print("0x%02X," % ord(c), end="")
+        n += 1
+        if n % 16 == 0:
+            print()
+print("};")
+
+print("const size_t " + symbol + "_len = sizeof(" + symbol + ");")

build/_sandbox.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python3
+
+from os.path import *
+import argparse
+import os
+import shutil
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-s", "--sandbox")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("-l", "--link", action="store_true")
+    parser.add_argument("-e", "--export", action="store_true")
+    parser.add_argument("files", nargs="*")
+    args = parser.parse_args()
+
+    assert args.sandbox, "You must specify a sandbox directory"
+    assert args.link ^ args.export, "You can't link and export at the same time"
+
+    if args.link:
+        os.makedirs(args.sandbox, exist_ok=True)
+        for f in args.files:
+            sf = join(args.sandbox, f)
+            if args.verbose:
+                print("link", sf)
+            os.makedirs(dirname(sf), exist_ok=True)
+            try:
+                os.symlink(abspath(f), sf)
+            except PermissionError:
+                shutil.copy(f, sf)
+
+    if args.export:
+        for f in args.files:
+            sf = join(args.sandbox, f)
+            if args.verbose:
+                print("export", sf)
+            df = dirname(f)
+            if df:
+                os.makedirs(df, exist_ok=True)
+
+            try:
+                os.remove(f)
+            except FileNotFoundError:
+                pass
+            os.rename(sf, f)
+
+
+main()

build/_zip.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+from os.path import *
+import argparse
+import os
+from zipfile import ZipFile
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-z", "--zipfile")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("-f", "--file", nargs=2, action="append")
+    args = parser.parse_args()
+
+    assert args.zipfile, "You must specify a zipfile to create"
+
+    with ZipFile(args.zipfile, mode="w") as zf:
+        for zipname, filename in args.file:
+            if args.verbose:
+                print(filename, "->", zipname)
+            zf.write(filename, arcname=zipname)
+
+
+main()

build/ab.mk

@@ -0,0 +1,119 @@
+MAKENOT4 := $(if $(findstring 3.9999, $(lastword $(sort 3.9999 $(MAKE_VERSION)))),yes,no)
+
+ifeq ($(MAKENOT4),yes)
+$(error You need GNU Make 4.x for this (if you're on OSX, use gmake).)
+endif
+
+OBJ ?= .obj
+PYTHON ?= python3
+PKG_CONFIG ?= pkg-config
+HOST_PKG_CONFIG ?= $(PKG_CONFIG)
+ECHO ?= echo
+CP ?= cp
+
+HOSTCC ?= gcc
+HOSTCXX ?= g++
+HOSTAR ?= ar
+HOSTCFLAGS ?= -g -Og
+HOSTCXXFLAGS ?= $(HOSTCFLAGS)
+HOSTLDFLAGS ?= -g
+
+CC ?= $(HOSTCC)
+CXX ?= $(HOSTCXX)
+AR ?= $(HOSTAR)
+CFLAGS ?= $(HOSTCFLAGS)
+CXXFLAGS ?= $(CFLAGS)
+LDFLAGS ?= $(HOSTLDFLAGS)
+
+NINJA ?= ninja
+
+ifdef VERBOSE
+	hide =
+else
+	ifdef V
+		hide =
+	else
+		hide = @
+	endif
+endif
+
+# If enabled, shows a nice display of how far through the build you are. This
+# doubles Make startup time. Also, on Make 4.3 and above, rebuilds don't show
+# correct progress information.
+AB_ENABLE_PROGRESS_INFO ?= true
+
+WINDOWS := no
+OSX := no
+LINUX := no
+ifeq ($(OS),Windows_NT)
+    WINDOWS := yes
+else
+    UNAME_S := $(shell uname -s)
+    ifeq ($(UNAME_S),Linux)
+		LINUX := yes
+    endif
+    ifeq ($(UNAME_S),Darwin)
+		OSX := yes
+    endif
+endif
+
+ifeq ($(OS), Windows_NT)
+	EXT ?= .exe
+endif
+EXT ?=
+
+CWD=$(shell pwd)
+
+define newline
+
+
+endef
+
+define check_for_command
+  $(shell command -v $1 >/dev/null || (echo "Required command '$1' missing" >/dev/stderr && kill $$PPID))
+endef
+
+$(call check_for_command,ninja)
+$(call check_for_command,cmp)
+$(call check_for_command,$(PYTHON))
+
+pkg-config-hash = $(shell ($(PKG_CONFIG) --list-all && $(HOST_PKG_CONFIG) --list-all) | md5sum)
+build-files = $(shell find . -name .obj -prune -o \( -name 'build.py' -a -type f \) -print) $(wildcard build/*.py) $(wildcard config.py)
+build-file-timestamps = $(shell ls -l $(build-files) | md5sum)
+
+# Wipe the build file (forcing a regeneration) if the make environment is different.
+# (Conveniently, this includes the pkg-config hash calculated above.)
+
+ignored-variables = MAKE_RESTARTS .VARIABLES MAKECMDGOALS MAKEFLAGS MFLAGS
+$(shell mkdir -p $(OBJ))
+$(file >$(OBJ)/newvars.txt,$(foreach v,$(filter-out $(ignored-variables),$(.VARIABLES)),$(v)=$($(v))$(newline)))
+$(shell touch $(OBJ)/vars.txt)
+#$(shell diff -u $(OBJ)/vars.txt $(OBJ)/newvars.txt > /dev/stderr)
+$(shell cmp -s $(OBJ)/newvars.txt $(OBJ)/vars.txt || (rm -f $(OBJ)/build.ninja && echo "Environment changed --- regenerating" > /dev/stderr))
+$(shell mv $(OBJ)/newvars.txt $(OBJ)/vars.txt)
+
+.PHONY: update-ab
+update-ab:
+	@echo "Press RETURN to update ab from the repository, or CTRL+C to cancel." \
+		&& read a \
+		&& (curl -L https://github.com/davidgiven/ab/releases/download/dev/distribution.tar.xz | tar xvJf -) \
+		&& echo "Done."
+
+.PHONY: clean
+clean::
+	@echo CLEAN
+	$(hide) rm -rf $(OBJ)
+
+export PYTHONHASHSEED = 1
+$(OBJ)/build.ninja $(OBJ)/build.targets &:
+	@echo "AB"
+	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py \
+		-o $(OBJ) build.py \
+		-v $(OBJ)/vars.txt \
+		|| (rm -f $@ && false)
+
+include $(OBJ)/build.targets
+.PHONY: $(ninja-targets)
+.NOTPARALLEL:
+$(ninja-targets): $(OBJ)/build.ninja
+	+$(hide) $(NINJA) -f $(OBJ)/build.ninja $@

build/ab.ninja

@@ -0,0 +1,2 @@
+rule rule
+    command = $command

build/ab.py

@@ -0,0 +1,725 @@
+from collections import namedtuple
+from copy import copy
+from importlib.machinery import SourceFileLoader, PathFinder, ModuleSpec
+from os.path import *
+from pathlib import Path
+from typing import Iterable
+import argparse
+import ast
+import builtins
+import functools
+import hashlib
+import importlib
+import importlib.util
+import inspect
+import os
+import re
+import string
+import sys
+import types
+
+VERBOSE_NINJA_FILE = False
+
+quiet = False
+cwdStack = [""]
+targets = {}
+unmaterialisedTargets = {}  # dict, not set, to get consistent ordering
+materialisingStack = []
+defaultGlobals = {}
+outputTargets = set()
+
+RE_FORMAT_SPEC = re.compile(
+    r"(?:(?P<fill>[\s\S])?(?P<align>[<>=^]))?"
+    r"(?P<sign>[- +])?"
+    r"(?P<pos_zero>z)?"
+    r"(?P<alt>#)?"
+    r"(?P<zero_padding>0)?"
+    r"(?P<width_str>\d+)?"
+    r"(?P<grouping>[_,])?"
+    r"(?:(?P<decimal>\.)(?P<precision_str>\d+))?"
+    r"(?P<type>[bcdeEfFgGnosxX%])?"
+)
+
+CommandFormatSpec = namedtuple(
+    "CommandFormatSpec", RE_FORMAT_SPEC.groupindex.keys()
+)
+
+sys.path += ["."]
+old_import = builtins.__import__
+
+
+class Environment(types.SimpleNamespace):
+    def setdefault(self, name, value):
+        if not hasattr(self, name):
+            setattr(self, name, value)
+
+
+G = Environment()
+
+
+class PathFinderImpl(PathFinder):
+    def find_spec(self, fullname, path, target=None):
+        # The second test here is needed for Python 3.9.
+        if not path or not path[0]:
+            path = ["."]
+        if len(path) != 1:
+            return None
+
+        try:
+            path = relpath(path[0])
+        except ValueError:
+            return None
+
+        realpath = fullname.replace(".", "/")
+        buildpath = realpath + ".py"
+        if isfile(buildpath):
+            spec = importlib.util.spec_from_file_location(
+                name=fullname,
+                location=buildpath,
+                loader=BuildFileLoaderImpl(fullname=fullname, path=buildpath),
+                submodule_search_locations=[],
+            )
+            return spec
+        if isdir(realpath):
+            return ModuleSpec(fullname, None, origin=realpath, is_package=True)
+        return None
+
+
+class BuildFileLoaderImpl(SourceFileLoader):
+    def exec_module(self, module):
+        sourcepath = relpath(module.__file__)
+
+        if not quiet:
+            print("loading", sourcepath)
+        cwdStack.append(dirname(sourcepath))
+        super(SourceFileLoader, self).exec_module(module)
+        cwdStack.pop()
+
+
+sys.meta_path.insert(0, PathFinderImpl())
+
+
+class ABException(BaseException):
+    pass
+
+
+def error(message):
+    raise ABException(message)
+
+
+def _undo_escaped_dollar(s, op):
+    return s.replace(f"$${op}", f"${op}")
+
+
+class BracketedFormatter(string.Formatter):
+    def parse(self, format_string):
+        while format_string:
+            m = re.search(f"(?:[^$]|^)()\\$\\[()", format_string)
+            if not m:
+                yield (
+                    _undo_escaped_dollar(format_string, "["),
+                    None,
+                    None,
+                    None,
+                )
+                break
+            left = format_string[: m.start(1)]
+            right = format_string[m.end(2) :]
+
+            offset = len(right) + 1
+            try:
+                ast.parse(right)
+            except SyntaxError as e:
+                if not str(e).startswith(f"unmatched ']'"):
+                    raise e
+                offset = e.offset
+
+            expr = right[0 : offset - 1]
+            format_string = right[offset:]
+
+            yield (
+                _undo_escaped_dollar(left, "[") if left else None,
+                expr,
+                None,
+                None,
+            )
+
+
+class GlobalFormatter(string.Formatter):
+    def parse(self, format_string):
+        while format_string:
+            m = re.search(f"(?:[^$]|^)()\\$\\(([^)]*)\\)()", format_string)
+            if not m:
+                yield (
+                    format_string,
+                    None,
+                    None,
+                    None,
+                )
+                break
+            left = format_string[: m.start(1)]
+            var = m[2]
+            format_string = format_string[m.end(3) :]
+
+            yield (
+                left if left else None,
+                var,
+                None,
+                None,
+            )
+
+    def get_field(self, name, a1, a2):
+        return (
+            getattr(G, name),
+            False,
+        )
+
+    def format_field(self, value, format_spec):
+        if not value:
+            return ""
+        return str(value)
+
+
+globalFormatter = GlobalFormatter()
+
+
+def substituteGlobalVariables(value):
+    while True:
+        oldValue = value
+        value = globalFormatter.format(value)
+        if value == oldValue:
+            return _undo_escaped_dollar(value, "(")
+
+
+def Rule(func):
+    sig = inspect.signature(func)
+
+    @functools.wraps(func)
+    def wrapper(*, name=None, replaces=None, **kwargs):
+        cwd = None
+        if "cwd" in kwargs:
+            cwd = kwargs["cwd"]
+            del kwargs["cwd"]
+
+        if not cwd:
+            if replaces:
+                cwd = replaces.cwd
+            else:
+                cwd = cwdStack[-1]
+
+        if name:
+            if name[0] != "+":
+                name = "+" + name
+            t = Target(cwd, join(cwd, name))
+
+            assert (
+                t.name not in targets
+            ), f"target {t.name} has already been defined"
+            targets[t.name] = t
+        elif replaces:
+            t = replaces
+        else:
+            raise ABException("you must supply either 'name' or 'replaces'")
+
+        t.cwd = cwd
+        t.types = func.__annotations__
+        t.callback = func
+        t.traits.add(func.__name__)
+        if "args" in kwargs:
+            t.explicit_args = kwargs["args"]
+            t.args.update(t.explicit_args)
+            del kwargs["args"]
+        if "traits" in kwargs:
+            t.traits |= kwargs["traits"]
+            del kwargs["traits"]
+
+        t.binding = sig.bind(name=name, self=t, **kwargs)
+        t.binding.apply_defaults()
+
+        unmaterialisedTargets[t] = None
+        if replaces:
+            t.materialise(replacing=True)
+        return t
+
+    defaultGlobals[func.__name__] = wrapper
+    return wrapper
+
+
+def _isiterable(xs):
+    return isinstance(xs, Iterable) and not isinstance(
+        xs, (str, bytes, bytearray)
+    )
+
+
+class Target:
+    def __init__(self, cwd, name):
+        self.name = name
+        self.localname = self.name.rsplit("+")[-1]
+        self.traits = set()
+        self.dir = join(G.OBJ, name)
+        self.ins = []
+        self.outs = []
+        self.deps = []
+        self.materialised = False
+        self.args = {}
+
+    def __eq__(self, other):
+        return self.name is other.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def __hash__(self):
+        return id(self)
+
+    def __repr__(self):
+        return f"Target('{self.name}')"
+
+    def templateexpand(selfi, s):
+        class Formatter(BracketedFormatter):
+            def get_field(self, name, a1, a2):
+                return (
+                    eval(name, selfi.callback.__globals__, selfi.args),
+                    False,
+                )
+
+            def format_field(self, value, format_spec):
+                if not value:
+                    return ""
+                if type(value) == str:
+                    return value
+                if _isiterable(value):
+                    value = list(value)
+                if type(value) != list:
+                    value = [value]
+                return " ".join(
+                    [selfi.templateexpand(f) for f in filenamesof(value)]
+                )
+
+        s = Formatter().format(s)
+        return substituteGlobalVariables(s)
+
+    def materialise(self, replacing=False):
+        if self not in unmaterialisedTargets:
+            return
+
+        if not replacing and self in materialisingStack:
+            print("Found dependency cycle:")
+            for i in materialisingStack:
+                print(f"  {i.name}")
+            print(f"  {self.name}")
+            sys.exit(1)
+        materialisingStack.append(self)
+
+        # Perform type conversion to the declared rule parameter types.
+
+        try:
+            for k, v in self.binding.arguments.items():
+                if k != "kwargs":
+                    t = self.types.get(k, None)
+                    if t:
+                        v = t.convert(v, self)
+                    self.args[k] = copy(v)
+                else:
+                    for kk, vv in v.items():
+                        t = self.types.get(kk, None)
+                        if t:
+                            vv = t.convert(v, self)
+                        self.args[kk] = copy(vv)
+            self.args["name"] = self.name
+            self.args["dir"] = self.dir
+            self.args["self"] = self
+
+            # Actually call the callback.
+
+            cwdStack.append(self.cwd)
+            if "kwargs" in self.binding.arguments.keys():
+                # If the caller wants kwargs, return all arguments except the standard ones.
+                cbargs = {
+                    k: v for k, v in self.args.items() if k not in {"dir"}
+                }
+            else:
+                # Otherwise, just call the callback with the ones it asks for.
+                cbargs = {}
+                for k in self.binding.arguments.keys():
+                    if k != "kwargs":
+                        try:
+                            cbargs[k] = self.args[k]
+                        except KeyError:
+                            error(
+                                f"invocation of {self} failed because {k} isn't an argument"
+                            )
+            self.callback(**cbargs)
+            cwdStack.pop()
+        except BaseException as e:
+            print(f"Error materialising {self}: {self.callback}")
+            print(f"Arguments: {self.args}")
+            raise e
+
+        if self.outs is None:
+            raise ABException(f"{self.name} didn't set self.outs")
+
+        if self in unmaterialisedTargets:
+            del unmaterialisedTargets[self]
+        materialisingStack.pop()
+        self.materialised = True
+
+    def convert(value, target):
+        if not value:
+            return None
+        return target.targetof(value)
+
+    def targetof(self, value):
+        if isinstance(value, str) and (value[0] == "="):
+            value = join(self.dir, value[1:])
+
+        return targetof(value, self.cwd)
+
+
+def _filetarget(value, cwd):
+    if value in targets:
+        return targets[value]
+
+    t = Target(cwd, value)
+    t.outs = [value]
+    targets[value] = t
+    return t
+
+
+def targetof(value, cwd=None):
+    if not cwd:
+        cwd = cwdStack[-1]
+    if isinstance(value, Path):
+        value = value.as_posix()
+    if isinstance(value, Target):
+        t = value
+    else:
+        assert (
+            value[0] != "="
+        ), "can only use = for targets associated with another target"
+
+        if value.startswith("."):
+            # Check for local rule.
+            if value.startswith(".+"):
+                value = normpath(join(cwd, value[1:]))
+            # Check for local path.
+            elif value.startswith("./"):
+                value = normpath(join(cwd, value))
+        # Explicit directories are always raw files.
+        if value.endswith("/"):
+            return _filetarget(value, cwd)
+        # Anything in .obj is a raw file.
+        elif value.startswith(outputdir) or value.startswith(G.OBJ):
+            return _filetarget(value, cwd)
+
+        # If this is not a rule lookup...
+        if "+" not in value:
+            # ...and if the value is pointing at a directory without a trailing /,
+            # it's a shorthand rule lookup.
+            if isdir(value):
+                value = value + "+" + basename(value)
+            # Otherwise it's an absolute file.
+            else:
+                return _filetarget(value, cwd)
+
+        # At this point we have the fully qualified name of a rule.
+
+        (path, target) = value.rsplit("+", 1)
+        value = join(path, "+" + target)
+        if value not in targets:
+            # Load the new build file.
+
+            path = join(path, "build.py")
+            try:
+                loadbuildfile(path)
+            except ModuleNotFoundError:
+                error(
+                    f"no such build file '{path}' while trying to resolve '{value}'"
+                )
+            assert (
+                value in targets
+            ), f"build file at '{path}' doesn't contain '+{target}' when trying to resolve '{value}'"
+
+        t = targets[value]
+
+    t.materialise()
+    return t
+
+
+class Targets:
+    def convert(value, target):
+        if not value:
+            return []
+        assert _isiterable(value), "cannot convert non-list to Targets"
+        return [target.targetof(x) for x in flatten(value)]
+
+
+class TargetsMap:
+    def convert(value, target):
+        if not value:
+            return {}
+        output = {k: target.targetof(v) for k, v in value.items()}
+        for k, v in output.items():
+            assert (
+                len(filenamesof([v])) == 1
+            ), f"targets of a TargetsMap used as an argument of {target} with key '{k}' must contain precisely one output file, but was {filenamesof([v])}"
+        return output
+
+
+def _removesuffix(self, suffix):
+    # suffix='' should not call self[:-0].
+    if suffix and self.endswith(suffix):
+        return self[: -len(suffix)]
+    else:
+        return self[:]
+
+
+def loadbuildfile(filename):
+    modulename = _removesuffix(filename.replace("/", "."), ".py")
+    if modulename not in sys.modules:
+        spec = importlib.util.spec_from_file_location(
+            name=modulename,
+            location=filename,
+            loader=BuildFileLoaderImpl(fullname=modulename, path=filename),
+            submodule_search_locations=[],
+        )
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[modulename] = module
+        spec.loader.exec_module(module)
+
+
+def flatten(items):
+    def generate(xs):
+        for x in xs:
+            if _isiterable(x):
+                yield from generate(x)
+            else:
+                yield x
+
+    return list(generate(items))
+
+
+def targetnamesof(items):
+    assert _isiterable(items), "argument of filenamesof is not a collection"
+
+    return [t.name for t in items]
+
+
+def filenamesof(items):
+    assert _isiterable(items), "argument of filenamesof is not a collection"
+
+    def generate(xs):
+        for x in xs:
+            if isinstance(x, Target):
+                x.materialise()
+                yield from generate(x.outs)
+            else:
+                yield x
+
+    return list(generate(items))
+
+
+def filenameof(x):
+    xs = filenamesof(x.outs)
+    assert (
+        len(xs) == 1
+    ), f"tried to use filenameof() on {x} which does not have exactly one output: {x.outs}"
+    return xs[0]
+
+
+def emit(*args, into=None):
+    s = " ".join(args) + "\n"
+    if into is not None:
+        into += [s]
+    else:
+        ninjaFp.write(s)
+
+
+def shell(*args):
+    s = "".join(args) + "\n"
+    shellFp.write(s)
+
+
+def emit_rule(self, ins, outs, cmds=[], label=None):
+    name = self.name
+    fins = [self.templateexpand(f) for f in set(filenamesof(ins))]
+    fouts = [self.templateexpand(f) for f in filenamesof(outs)]
+
+    global outputTargets
+    outputTargets.update(fouts)
+    outputTargets.add(name)
+
+    emit("")
+    if VERBOSE_NINJA_FILE:
+        for k, v in self.args.items():
+            emit(f"# {k} = {v}")
+
+    if outs:
+        os.makedirs(self.dir, exist_ok=True)
+        rule = []
+
+        sandbox = join(self.dir, "sandbox")
+        emit(f"rm -rf {sandbox}", into=rule)
+        emit(
+            f"{G.PYTHON} build/_sandbox.py --link -s", sandbox, *fins, into=rule
+        )
+        for c in cmds:
+            emit(f"(cd {sandbox} &&", c, ")", into=rule)
+        emit(
+            f"{G.PYTHON} build/_sandbox.py --export -s",
+            sandbox,
+            *fouts,
+            into=rule,
+        )
+
+        ruletext = "".join(rule)
+        if len(ruletext) > 7000:
+            rulehash = hashlib.sha1(ruletext.encode()).hexdigest()
+
+            rulef = join(self.dir, f"rule-{rulehash}.sh")
+            with open(rulef, "wt") as fp:
+                fp.write("set -e\n")
+                fp.write(ruletext)
+
+            emit("build", *fouts, ":rule", *fins, rulef)
+            emit(" command=sh", rulef)
+        else:
+            emit("build", *fouts, ":rule", *fins)
+            emit(
+                " command=",
+                "&&".join([s.strip() for s in rule]).replace("$", "$$"),
+            )
+        if label:
+            emit(" description=", label)
+        emit("build", name, ":phony", *fouts)
+
+    else:
+        assert len(cmds) == 0, "rules with no outputs cannot have commands"
+        emit("build", name, ":phony", *fins)
+
+    emit("")
+
+
+@Rule
+def simplerule(
+    self,
+    name,
+    ins: Targets = [],
+    outs: Targets = [],
+    deps: Targets = [],
+    commands=[],
+    label="RULE",
+):
+    self.ins = ins
+    self.outs = outs
+    self.deps = deps
+
+    dirs = []
+    cs = []
+    for out in filenamesof(outs):
+        dir = dirname(out)
+        if dir and dir not in dirs:
+            dirs += [dir]
+
+        cs = [("mkdir -p %s" % dir) for dir in dirs]
+
+    for c in commands:
+        cs += [self.templateexpand(c)]
+
+    emit_rule(
+        self=self,
+        ins=ins + deps,
+        outs=outs,
+        label=self.templateexpand("$[label] $[name]") if label else None,
+        cmds=cs,
+    )
+
+
+@Rule
+def export(self, name=None, items: TargetsMap = {}, deps: Targets = []):
+    ins = []
+    outs = []
+    for dest, src in items.items():
+        dest = self.targetof(dest)
+        outs += [dest]
+
+        destf = self.templateexpand(filenameof(dest))
+        outputTargets.update([destf])
+
+        srcs = filenamesof([src])
+        assert (
+            len(srcs) == 1
+        ), "a dependency of an exported file must have exactly one output file"
+        srcf = self.templateexpand(srcs[0])
+
+        subrule = simplerule(
+            name=f"{self.localname}/{destf}",
+            cwd=self.cwd,
+            ins=[srcs[0]],
+            outs=[destf],
+            commands=["$(CP) -H %s %s" % (srcf, destf)],
+            label="EXPORT",
+        )
+        subrule.materialise()
+
+    self.ins = []
+    self.outs = deps + outs
+    outputTargets.add(name)
+
+    emit("")
+    emit(
+        "build",
+        name,
+        ":phony",
+        *[self.templateexpand(f) for f in filenamesof(outs + deps)],
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-q", "--quiet", action="store_true")
+    parser.add_argument("-v", "--varfile")
+    parser.add_argument("-o", "--outputdir")
+    parser.add_argument("-D", "--define", action="append", default=[])
+    parser.add_argument("files", nargs="+")
+    args = parser.parse_args()
+
+    global quiet
+    quiet = args.quiet
+
+    vardefs = args.define
+    if args.varfile:
+        with open(args.varfile, "rt") as fp:
+            vardefs = vardefs + list(fp)
+
+    for line in vardefs:
+        if "=" in line:
+            name, value = line.split("=", 1)
+            G.setdefault(name.strip(), value.strip())
+
+    global ninjaFp, shellFp, outputdir
+    outputdir = args.outputdir
+    G.setdefault("OBJ", outputdir)
+    ninjaFp = open(outputdir + "/build.ninja", "wt")
+    ninjaFp.write(f"include build/ab.ninja\n")
+
+    for k in ["Rule"]:
+        defaultGlobals[k] = globals()[k]
+
+    global __name__
+    sys.modules["build.ab"] = sys.modules[__name__]
+    __name__ = "build.ab"
+
+    for f in args.files:
+        loadbuildfile(f)
+
+    while unmaterialisedTargets:
+        t = next(iter(unmaterialisedTargets))
+        t.materialise()
+
+    with open(outputdir + "/build.targets", "wt") as fp:
+        fp.write("ninja-targets =")
+        fp.write(substituteGlobalVariables(" ".join(outputTargets)))
+
+
+main()

build/build.lua

@@ -1,251 +0,0 @@
-local OBJDIR = "$(OBJDIR)"
-
-local function objdir(e)
-	return concatpath(OBJDIR, e.cwd, e.name)
-end
-
-definerule("normalrule",
-	{
-		ins = { type="targets" },
-		deps = { type="targets", default={} },
-		outs = { type="targets", default={} },
-		outleaves = { type="strings" },
-		label = { type="string", optional=true },
-		objdir = { type="string", optional=true },
-		commands = { type="strings" },
-	},
-	function (e)
-		local dir = e.objdir or objdir(e)
-		local realouts = {}
-		for _, v in pairs(e.outleaves) do
-			realouts[#realouts+1] = concatpath(dir, v)
-		end
-
-		local vars = inherit(e.vars, {
-			dir = dir
-		})
-
-		local result = simplerule {
-			name = e.name,
-			ins = e.ins,
-			deps = e.deps,
-			outs = concat(realouts, filenamesof(e.outs)),
-			label = e.label,
-			commands = e.commands,
-			vars = vars,
-		}
-		result.dir = dir
-		return result
-	end
-)
-
-local function is_clike(f)
-	return f:find("%.c$") or f:find("%.cc$") or f:find("%.cpp$")
-end
-
-definerule("cfile",
-	{
-		srcs = { type="targets" },
-		deps = { type="targets", default={} }
-	},
-	function (e)
-		local cflags = e.vars.cflags
-		local cxxflags = e.vars.cxxflags
-		for _, target in ipairs(targetsof(e.deps)) do
-			if target.is.clibrary then
-				cflags = concat(cflags, target.dep_cflags)
-				cxxflags = concat(cxxflags, target.dep_cxxflags)
-			end
-		end
-
-		local src = filter(filenamesof(e.srcs), is_clike)
-		local cmd
-		local cxx = false
-		if src[1]:find("%.c$") then
-			cmd = "$(CC) -c -o %{outs[1]} %{ins[1]} %{hdrpaths} %{cflags}"
-		else
-			cmd = "$(CXX) -c -o %{outs[1]} %{ins[1]} %{hdrpaths} %{cflags} %{cxxflags}"
-			cxx = true
-		end
-
-		local outleaf = basename(e.name)..".o"
-		local rule = normalrule {
-			name = e.name,
-			cwd = e.cwd,
-			ins = e.srcs,
-			deps = e.deps,
-			outleaves = {outleaf},
-			label = e.label,
-			commands = cmd,
-			vars = {
-				hdrpaths = {},
-				cflags = cflags,
-				cxxflags = cxxflags,
-			}
-		}
-
-		rule.is.cxxfile = cxx
-		return rule
-	end
-)
-
-local function do_cfiles(e)
-	local outs = {}
-	local srcs = filenamesof(e.srcs)
-	for _, f in ipairs(sorted(filter(srcs, is_clike))) do
-		local ofile
-		if f:find(OBJDIR, 1, true) == 1 then
-			ofile = e.name.."/"..f:sub(#OBJDIR+1)..".o"
-		else
-			ofile = e.name.."/"..f..".o"
-		end
-		outs[#outs+1] = cfile {
-			name = ofile,
-			srcs = { f },
-			deps = e.deps
-		}
-	end
-	return outs
-end
-
-definerule("clibrary",
-	{
-		srcs = { type="targets", default={} },
-		deps = { type="targets", default={} },
-		hdrs = { type="targets", default={} },
-		dep_cflags = { type="strings", default={} },
-		dep_cxxflags = { type="strings", default={} },
-		dep_ldflags = { type="strings", default={} },
-		dep_libs = { type="strings", default={} },
-	},
-	function (e)
-		local ins = do_cfiles(e)
-		local cxx = false
-		for _, f in ipairs(ins) do
-			if f.is.cxxfile then
-				cxx = true
-				break
-			end
-		end
-
-		local mkdirs = {}
-		local copies = {}
-		local outs = {}
-		local function copy_file(src, dest)
-			mkdirs[#mkdirs+1] = "mkdir -p %{dir}/"..dirname(dest)
-			copies[#copies+1] = "cp "..src.." %{dir}/"..dest
-			outs[#outs+1] = objdir(e).."/"..dest
-		end
-
-		local deps = {}
-		for k, v in pairs(e.hdrs) do
-			deps[#deps+1] = v
-			if type(k) == "number" then
-				v = filenamesof(v)
-				for _, v in ipairs(v) do
-					if not startswith(e.cwd, v) then
-						error(string.format("filename '%s' is not local to '%s' --- "..
-							"you'll have to specify the output filename manually", v, e.cwd))
-					end
-					copy_file(v, v:gsub("^"..e.cwd, ""))
-				end
-			else
-				v = filenamesof(v)
-				if #v ~= 1 then
-					error("each mapped hdrs item can only cope with a single file")
-				end
-				copy_file(v[1], k)
-			end
-		end
-
-		ins = sorted(filenamesof(ins))
-		local has_ar = (#ins ~= 0)
-		local lib = normalrule {
-			name = e.name,
-			cwd = e.cwd,
-			ins = sorted(filenamesof(ins)),
-			deps = deps,
-			outs = outs,
-			outleaves = { e.name..".a" },
-			label = e.label,
-			commands = {
-				sorted(mkdirs),
-				sorted(copies),
-				has_ar and "rm -f %{outs[1]} && $(AR) cqs %{outs[1]} %{ins}" or {},
-			}
-		}
-		
-		lib.dep_cflags = concat(e.dep_cflags, "-I"..lib.dir)
-		lib.dep_cxxflags = e.dep_cxxflags
-		lib.dep_ldflags = e.dep_ldflags
-		lib.dep_libs = concat(e.dep_libs, has_ar and matching(filenamesof(lib), "%.a$") or {})
-		lib.dep_cxx = cxx
-
-		for _, d in pairs(targetsof(e.deps)) do
-			lib.dep_cflags = concat(lib.dep_cflags, d.dep_cflags)
-			lib.dep_cxxflags = concat(lib.dep_cxxflags, d.dep_cxxflags)
-			lib.dep_ldflags = concat(lib.dep_ldflags, d.dep_ldflags)
-			lib.dep_libs = concat(lib.dep_libs, d.dep_libs)
-			lib.dep_cxx = lib.dep_cxx or d.dep_cxx
-		end
-
-		return lib
-	end
-)
-
-definerule("cprogram",
-	{
-		srcs = { type="targets", default={} },
-		deps = { type="targets", default={} },
-	},
-	function (e)
-		local deps = e.deps
-		local ins = {}
-		local cxx = false
-
-		if (#e.srcs > 0) then
-			local objs = do_cfiles(e)
-			for _, obj in pairs(objs) do
-				if obj.is.cxxfile then
-					cxx = true
-				end
-				ins[#ins+1] = obj
-			end
-		end
-
-		local libs = {}
-		local cflags = {}
-		local cxxflags = {}
-		local ldflags = {}
-		for _, lib in pairs(e.deps) do
-			cflags = concat(cflags, lib.dep_cflags)
-			cxxflags = concat(cxxflags, lib.dep_cxxflags)
-			ldflags = concat(ldflags, lib.dep_ldflags)
-			libs = concat(libs, lib.dep_libs)
-			cxx = cxx or lib.dep_cxx
-		end
-
-		local command
-		if cxx then
-			command = "$(CXX) $(LDFLAGS) %{ldflags} -o %{outs[1]} %{ins} %{libs} %{libs}"
-		else
-			command = "$(CC) $(LDFLAGS) %{ldflags} -o %{outs[1]} %{ins} %{libs} %{libs}"
-		end
-
-		return normalrule {
-			name = e.name,
-			cwd = e.cwd,
-			deps = deps,
-			ins = ins,
-			outleaves = { e.name },
-			commands = { command },
-			vars = {
-				cflags = cflags,
-				cxxflags = cxxflags,
-				ldflags = ldflags,
-				libs = libs,
-			}
-		}
-	end
-)
-

build/c.py

@@ -0,0 +1,580 @@
+from build.ab import (
+    Rule,
+    Targets,
+    TargetsMap,
+    filenameof,
+    filenamesof,
+    flatten,
+    simplerule,
+    emit,
+    G,
+)
+from build.utils import stripext, collectattrs
+from build.toolchain import Toolchain, HostToolchain
+from os.path import *
+
+if G.OSX != "yes":
+    G.STARTGROUP = "-Wl,--start-group"
+    G.ENDGROUP = "-Wl,--end-group"
+else:
+    G.STARTGROUP = ""
+    G.ENDGROUP = ""
+
+Toolchain.CC = ["$(CC) -c -o $[outs[0]] $[ins[0]] $(CFLAGS) $[cflags]"]
+Toolchain.CPP = ["$(CC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
+Toolchain.CXX = ["$(CXX) -c -o $[outs[0]] $[ins[0]] $(CXXFLAGS) $[cflags]"]
+Toolchain.AR = ["$(AR) cqs $[outs[0]] $[ins]"]
+Toolchain.ARXX = ["$(AR) cqs $[outs[0]] $[ins]"]
+Toolchain.CLINK = [
+    "$(CC) -o $[outs[0]] $(STARTGROUP) $[ins] $[ldflags] $(LDFLAGS) $(ENDGROUP)"
+]
+Toolchain.CXXLINK = [
+    "$(CXX) -o $[outs[0]] $(STARTGROUP) $[ins] $[ldflags] $(LDFLAGS) $(ENDGROUP)"
+]
+
+Toolchain.is_source_file = (
+    lambda f: f.endswith(".c")
+    or f.endswith(".cc")
+    or f.endswith(".cpp")
+    or f.endswith(".S")
+    or f.endswith(".s")
+    or f.endswith(".m")
+    or f.endswith(".mm")
+)
+
+
+# Given a set of dependencies, finds the set of relevant library targets (i.e.
+# contributes *.a files) for compiling C programs.  The actual list of libraries
+# is in dep.clibrary_files.
+def _toolchain_find_library_targets(deps):
+    lib_deps = []
+    for d in deps:
+        lib_deps = _combine(lib_deps, d.args.get("clibrary_deps", []))
+    return lib_deps
+
+
+Toolchain.find_c_library_targets = _toolchain_find_library_targets
+
+
+# Given a set of dependencies, finds the set of relevant header targets (i.e.
+# contributes *.h files) for compiling C programs.  The actual list of libraries
+# is in dep.cheader_files.
+def _toolchain_find_header_targets(deps, initial=[]):
+    hdr_deps = initial
+    for d in deps:
+        hdr_deps = _combine(hdr_deps, d.args.get("cheader_deps", []))
+    return hdr_deps
+
+
+Toolchain.find_c_header_targets = _toolchain_find_header_targets
+
+
+HostToolchain.CC = ["$(HOSTCC) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
+HostToolchain.CPP = ["$(HOSTCC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
+HostToolchain.CXX = ["$(HOSTCXX) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
+HostToolchain.AR = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
+HostToolchain.ARXX = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
+HostToolchain.CLINK = [
+    "$(HOSTCC) -o $[outs[0]] $(STARTGROUP) $[ins] $[ldflags] $(HOSTLDFLAGS) $(ENDGROUP)"
+]
+HostToolchain.CXXLINK = [
+    "$(HOSTCXX) -o $[outs[0]] $(STARTGROUP) $[ins] $[ldflags] $(HOSTLDFLAGS) $(ENDGROUP)"
+]
+
+
+def _combine(list1, list2):
+    r = list(list1)
+    for i in list2:
+        if i not in r:
+            r.append(i)
+    return r
+
+
+def _indirect(deps, name):
+    r = []
+    for d in deps:
+        r = _combine(r, d.args.get(name, [d]))
+    return r
+
+
+def cfileimpl(self, name, srcs, deps, suffix, commands, label, toolchain, cflags):
+    outleaf = "=" + stripext(basename(filenameof(srcs[0]))) + suffix
+
+    hdr_deps = toolchain.find_c_header_targets(deps)
+    other_deps = [
+        d
+        for d in deps
+        if ("cheader_deps" not in d.args) and ("clibrary_deps" not in d.args)
+    ]
+    hdr_files = collectattrs(targets=hdr_deps, name="cheader_files")
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
+
+    t = simplerule(
+        replaces=self,
+        ins=srcs,
+        deps=other_deps + hdr_files,
+        outs=[outleaf],
+        label=label,
+        commands=commands,
+        args={"cflags": cflags},
+    )
+
+
+@Rule
+def cfile(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    suffix=".o",
+    toolchain=Toolchain,
+    label="CC",
+):
+    cfileimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        suffix,
+        toolchain.CC,
+        toolchain.PREFIX + label,
+        toolchain,
+        cflags,
+    )
+
+
+@Rule
+def cxxfile(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    suffix=".o",
+    toolchain=Toolchain,
+    label="CXX",
+):
+    cfileimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        suffix,
+        toolchain.CXX,
+        toolchain.PREFIX + label,
+        toolchain,
+        cflags,
+    )
+
+
+def _removeprefix(self, prefix):
+    if self.startswith(prefix):
+        return self[len(prefix) :]
+    else:
+        return self[:]
+
+
+def findsources(self, srcs, deps, cflags, filerule, toolchain, cwd):
+    for f in filenamesof(srcs):
+        if not toolchain.is_source_file(f):
+            cflags = cflags + [f"-I{dirname(f)}"]
+            deps = deps + [f]
+
+    objs = []
+    for s in flatten(srcs):
+        objs += [
+            filerule(
+                name=join(self.localname, _removeprefix(f, G.OBJ + "/")),
+                srcs=[f],
+                deps=deps,
+                cflags=sorted(set(cflags)),
+                toolchain=toolchain,
+                cwd=cwd,
+                args=getattr(self, "explicit_args", {}),
+            )
+            for f in filenamesof([s])
+            if toolchain.is_source_file(f)
+        ]
+        if any(f.endswith(".o") for f in filenamesof([s])):
+            objs += [s]
+
+    return objs
+
+
+def libraryimpl(
+    self,
+    name,
+    srcs,
+    deps,
+    hdrs,
+    caller_cflags,
+    caller_ldflags,
+    cflags,
+    ldflags,
+    toolchain,
+    commands,
+    label,
+    filerule,
+):
+    hdr_deps = toolchain.find_c_header_targets(deps) + [self]
+    lib_deps = toolchain.find_c_library_targets(deps) + [self]
+
+    hr = None
+    hf = []
+    ar = None
+    if hdrs:
+        cs = []
+        ins = hdrs.values()
+        outs = []
+        i = 0
+        for dest, src in hdrs.items():
+            s = filenamesof([src])
+            assert len(s) == 1, "the target of a header must return exactly one file"
+
+            cs += [f"$(CP) $[ins[{i}]] $[outs[{i}]]"]
+            outs += ["=" + dest]
+            i = i + 1
+
+        hr = simplerule(
+            name=f"{self.localname}_hdr",
+            ins=ins,
+            outs=outs,
+            commands=cs,
+            label=toolchain.PREFIX + "CHEADERS",
+        )
+        hr.args["cheader_deps"] = [hr]
+        hr.args["cheader_files"] = [hr]
+        hf = [f"-I{hr.dir}"]
+
+    if srcs:
+        # Can't depend on the current target to get the library headers, because
+        # if we do it'll cause a dependency loop.
+        objs = findsources(
+            self,
+            srcs,
+            deps + ([hr] if hr else []),
+            cflags + hf,
+            filerule,
+            toolchain,
+            self.cwd,
+        )
+
+        ar = simplerule(
+            name=f"{self.localname}_lib",
+            ins=objs,
+            outs=[f"={self.localname}.a"],
+            deps=deps,
+            label=label,
+            commands=commands,
+        )
+        ar.materialise()
+
+    self.outs = ([hr] if hr else []) + ([ar] if ar else [])
+    self.deps = self.outs
+    self.args["cheader_deps"] = hdr_deps
+    self.args["clibrary_deps"] = lib_deps
+    self.args["cheader_files"] = [hr] if hr else []
+    self.args["clibrary_files"] = [ar] if ar else []
+    self.args["caller_cflags"] = caller_cflags + hf
+    self.args["caller_ldflags"] = caller_ldflags
+
+
+@Rule
+def clibrary(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    hdrs: TargetsMap = None,
+    caller_cflags=[],
+    caller_ldflags=[],
+    cflags=[],
+    ldflags=[],
+    toolchain=Toolchain,
+    label="LIB",
+    cfilerule=cfile,
+):
+    libraryimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        hdrs,
+        caller_cflags,
+        caller_ldflags,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.AR,
+        toolchain.PREFIX + label,
+        cfilerule,
+    )
+
+
+@Rule
+def hostclibrary(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    hdrs: TargetsMap = None,
+    caller_cflags=[],
+    caller_ldflags=[],
+    cflags=[],
+    ldflags=[],
+    toolchain=HostToolchain,
+    label="LIB",
+    cfilerule=cfile,
+):
+    libraryimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        hdrs,
+        caller_cflags,
+        caller_ldflags,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.AR,
+        toolchain.PREFIX + label,
+        cfilerule,
+    )
+
+
+@Rule
+def cxxlibrary(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    hdrs: TargetsMap = None,
+    caller_cflags=[],
+    caller_ldflags=[],
+    cflags=[],
+    ldflags=[],
+    toolchain=Toolchain,
+    label="CXXLIB",
+    cxxfilerule=cxxfile,
+):
+    libraryimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        hdrs,
+        caller_cflags,
+        caller_ldflags,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.ARXX,
+        toolchain.PREFIX + label,
+        cxxfilerule,
+    )
+
+
+@Rule
+def hostcxxlibrary(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    hdrs: TargetsMap = None,
+    caller_cflags=[],
+    caller_ldflags=[],
+    cflags=[],
+    ldflags=[],
+    toolchain=HostToolchain,
+    label="CXXLIB",
+    cxxfilerule=cxxfile,
+):
+    libraryimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        hdrs,
+        caller_cflags,
+        caller_ldflags,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.ARXX,
+        toolchain.PREFIX + label,
+        cxxfilerule,
+    )
+
+
+def programimpl(
+    self,
+    name,
+    srcs,
+    deps,
+    cflags,
+    ldflags,
+    toolchain,
+    commands,
+    label,
+    filerule,
+):
+    cfiles = findsources(self, srcs, deps, cflags, filerule, toolchain, self.cwd)
+
+    lib_deps = toolchain.find_c_library_targets(deps)
+    libs = collectattrs(targets=lib_deps, name="clibrary_files")
+    ldflags = collectattrs(targets=lib_deps, name="caller_ldflags", initial=ldflags)
+
+    simplerule(
+        replaces=self,
+        ins=cfiles + libs,
+        outs=[f"={self.localname}{toolchain.EXE}"],
+        deps=deps,
+        label=label,
+        commands=commands,
+        args={"ldflags": ldflags},
+    )
+
+
+@Rule
+def cprogram(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    ldflags=[],
+    toolchain=Toolchain,
+    label="CLINK",
+    cfilerule=cfile,
+):
+    programimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.CLINK,
+        toolchain.PREFIX + label,
+        cfilerule,
+    )
+
+
+@Rule
+def hostcprogram(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    ldflags=[],
+    toolchain=HostToolchain,
+    label="CLINK",
+    cfilerule=cfile,
+):
+    programimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.CLINK,
+        toolchain.PREFIX + label,
+        cfilerule,
+    )
+
+
+@Rule
+def cxxprogram(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    ldflags=[],
+    toolchain=Toolchain,
+    label="CXXLINK",
+    cxxfilerule=cxxfile,
+):
+    programimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.CXXLINK,
+        toolchain.PREFIX + label,
+        cxxfilerule,
+    )
+
+
+@Rule
+def hostcxxprogram(
+    self,
+    name,
+    srcs: Targets = None,
+    deps: Targets = None,
+    cflags=[],
+    ldflags=[],
+    toolchain=HostToolchain,
+    label="CXXLINK",
+    cxxfilerule=cxxfile,
+):
+    programimpl(
+        self,
+        name,
+        srcs,
+        deps,
+        cflags,
+        ldflags,
+        toolchain,
+        toolchain.CXXLINK,
+        toolchain.PREFIX + label,
+        cxxfilerule,
+    )
+
+
+def _cppfileimpl(self, name, srcs, deps, cflags, toolchain):
+    hdr_deps = _indirect(deps, "cheader_deps")
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
+
+    simplerule(
+        replaces=self,
+        ins=srcs,
+        outs=[f"={self.localname}"],
+        deps=deps,
+        commands=toolchain.CPP,
+        args={"cflags": cflags},
+        label=toolchain.PREFIX + "CPPFILE",
+    )
+
+
+@Rule
+def cppfile(
+    self,
+    name,
+    srcs: Targets = [],
+    deps: Targets = [],
+    cflags=[],
+    toolchain=Toolchain,
+):
+    _cppfileimpl(self, name, srcs, deps, cflags, toolchain)
+
+
+@Rule
+def hostcppfile(
+    self,
+    name,
+    srcs: Targets = [],
+    deps: Targets = [],
+    cflags=[],
+    toolchain=HostToolchain,
+):
+    _cppfileimpl(self, name, srcs, deps, cflags, toolchain)

build/pkg.py

@@ -0,0 +1,83 @@
+from build.ab import Rule, Target, G
+import os
+import subprocess
+
+
+class _PkgConfig:
+    package_present = set()
+    package_properties = {}
+    pkgconfig = None
+
+    def __init__(self, cmd):
+        assert cmd, "no pkg-config environment variable supplied"
+        self.pkgconfig = cmd
+
+        r = subprocess.run(f"{cmd} --list-all", shell=True, capture_output=True)
+        ps = r.stdout.decode("utf-8")
+        self.package_present = {l.split(" ", 1)[0] for l in ps.splitlines()}
+
+    def has_package(self, name):
+        return name in self.package_present
+
+    def get_property(self, name, flag):
+        p = f"{name}.{flag}"
+        if p not in self.package_properties:
+            r = subprocess.run(
+                f"{self.pkgconfig} {flag} {name}",
+                shell=True,
+                capture_output=True,
+            )
+            self.package_properties[p] = r.stdout.decode("utf-8").strip()
+        return self.package_properties[p]
+
+
+TargetPkgConfig = _PkgConfig(G.PKG_CONFIG)
+HostPkgConfig = _PkgConfig(G.HOST_PKG_CONFIG)
+
+
+def _package(self, name, package, fallback, pkgconfig):
+    if pkgconfig.has_package(package):
+        cflags = pkgconfig.get_property(package, "--cflags")
+        ldflags = pkgconfig.get_property(package, "--libs")
+
+        if cflags:
+            self.args["caller_cflags"] = [cflags]
+        if ldflags:
+            self.args["caller_ldflags"] = [ldflags]
+        self.args["clibrary_deps"] = [self]
+        self.args["cheader_deps"] = [self]
+        self.traits.update({"clibrary", "cxxlibrary"})
+        return
+
+    assert fallback, f"Required package '{package}' not installed"
+
+    if "cheader_deps" in fallback.args:
+        self.args["cheader_deps"] = fallback.args["cheader_deps"]
+    if "clibrary_deps" in fallback.args:
+        self.args["clibrary_deps"] = fallback.args["clibrary_deps"]
+    if "cheader_files" in fallback.args:
+        self.args["cheader_files"] = fallback.args["cheader_files"]
+    if "clibrary_files" in fallback.args:
+        self.args["clibrary_files"] = fallback.args["clibrary_files"]
+    self.ins = fallback.ins
+    self.outs = fallback.outs
+    self.deps = fallback.deps
+    self.traits = fallback.traits
+
+
+@Rule
+def package(self, name, package=None, fallback: Target = None):
+    _package(self, name, package, fallback, TargetPkgConfig)
+
+
+@Rule
+def hostpackage(self, name, package=None, fallback: Target = None):
+    _package(self, name, package, fallback, HostPkgConfig)
+
+
+def has_package(name):
+    return TargetPkgConfig.has_package(name)
+
+
+def has_host_package(name):
+    return HostPkgConfig.has_package(name)

build/protobuf.py

@@ -0,0 +1,180 @@
+from build.ab import Rule, Targets, emit, simplerule, filenamesof, G
+from build.utils import filenamesmatchingof, collectattrs
+from os.path import join, abspath, dirname, relpath
+from build.pkg import has_package
+
+G.setdefault("PROTOC", "protoc")
+G.setdefault("HOSTPROTOC", "hostprotoc")
+
+assert has_package("protobuf"), "required package 'protobuf' not installed"
+
+
+def _getprotodeps(deps):
+    r = set()
+    for d in deps:
+        r.update(d.args.get("protodeps", {d}))
+    return sorted(r)
+
+
+@Rule
+def proto(self, name, srcs: Targets = [], deps: Targets = []):
+    protodeps = _getprotodeps(deps)
+    descriptorlist = ":".join(
+        [
+            relpath(f, start=self.dir)
+            for f in filenamesmatchingof(protodeps, "*.descriptor")
+        ]
+    )
+
+    dirs = sorted({"$[dir]/" + dirname(f) for f in filenamesof(srcs)})
+    simplerule(
+        replaces=self,
+        ins=srcs,
+        outs=[f"={self.localname}.descriptor"],
+        deps=protodeps,
+        commands=(
+            ["mkdir -p " + (" ".join(dirs))]
+            + [f"$(CP) {f} $[dir]/{f}" for f in filenamesof(srcs)]
+            + [
+                "cd $[dir] && "
+                + (
+                    " ".join(
+                        [
+                            "$(PROTOC)",
+                            "--proto_path=.",
+                            "--include_source_info",
+                            f"--descriptor_set_out={self.localname}.descriptor",
+                        ]
+                        + (
+                            [f"--descriptor_set_in={descriptorlist}"]
+                            if descriptorlist
+                            else []
+                        )
+                        + ["$[ins]"]
+                    )
+                )
+            ]
+        ),
+        label="PROTO",
+        args={
+            "protosrcs": filenamesof(srcs),
+            "protodeps": set(protodeps) | {self},
+        },
+    )
+
+
+@Rule
+def protolib(self, name, srcs: Targets = []):
+    simplerule(
+        replaces=self,
+        label="PROTOLIB",
+        args={
+            "protosrcs": collectattrs(targets=srcs, name="protosrcs"),
+            "protodeps": set(_getprotodeps(srcs)),
+        },
+    )
+
+
+@Rule
+def protocc(self, name, srcs: Targets = [], deps: Targets = []):
+    outs = []
+    protos = []
+
+    allsrcs = collectattrs(targets=srcs, name="protosrcs")
+    assert allsrcs, "no sources provided"
+    for f in filenamesmatchingof(allsrcs, "*.proto"):
+        cc = f.replace(".proto", ".pb.cc")
+        h = f.replace(".proto", ".pb.h")
+        protos += [f]
+        outs += ["=" + cc, "=" + h]
+
+    protodeps = _getprotodeps(deps + srcs)
+    descriptorlist = ":".join(
+        [
+            relpath(f, start=self.dir)
+            for f in filenamesmatchingof(protodeps, "*.descriptor")
+        ]
+    )
+
+    r = simplerule(
+        name=f"{self.localname}_srcs",
+        cwd=self.cwd,
+        ins=srcs,
+        outs=outs,
+        deps=protodeps,
+        commands=[
+            "cd $[dir] && "
+            + (
+                " ".join(
+                    [
+                        "$(PROTOC)",
+                        "--proto_path=.",
+                        "--cpp_out=.",
+                        f"--descriptor_set_in={descriptorlist}",
+                    ]
+                    + protos
+                )
+            )
+        ],
+        label="PROTOCC",
+    )
+
+    headers = {f[1:]: join(r.dir, f[1:]) for f in outs if f.endswith(".pb.h")}
+
+    from build.c import cxxlibrary
+
+    cxxlibrary(
+        replaces=self,
+        srcs=[r],
+        deps=deps,
+        hdrs=headers,
+    )
+
+
+@Rule
+def protojava(self, name, srcs: Targets = [], deps: Targets = []):
+    outs = []
+
+    allsrcs = collectattrs(targets=srcs, name="protosrcs")
+    assert allsrcs, "no sources provided"
+    protos = []
+    for f in filenamesmatchingof(allsrcs, "*.proto"):
+        protos += [f]
+        srcs += [f]
+
+    descriptorlist = ":".join(
+        [abspath(f) for f in filenamesmatchingof(srcs + deps, "*.descriptor")]
+    )
+
+    r = simplerule(
+        name=f"{self.localname}_srcs",
+        cwd=self.cwd,
+        ins=protos,
+        outs=[f"={self.localname}.srcjar"],
+        deps=srcs + deps,
+        commands=[
+            "mkdir -p $[dir]/srcs",
+            "cd $[dir]/srcs && "
+            + (
+                " ".join(
+                    [
+                        "$(PROTOC)",
+                        "--proto_path=.",
+                        "--java_out=.",
+                        f"--descriptor_set_in={descriptorlist}",
+                    ]
+                    + protos
+                )
+            ),
+            "$(JAR) cf $[outs[0]] -C $[dir]/srcs .",
+        ],
+        traits={"srcjar"},
+        label="PROTOJAVA",
+    )
+
+    from build.java import javalibrary
+
+    javalibrary(
+        replaces=self,
+        deps=[r] + deps,
+    )

build/tests.lua

@@ -1,18 +0,0 @@
-definerule("test",
-	{
-		srcs = { type="targets", default={} },
-	},
-	function (e)
-		if vars.TESTS == "yes" then
-			normalrule {
-				name = e.name,
-				ins = e.srcs,
-				outleaves = { "log.txt" },
-				commands = {
-					"%{ins} > %{outs}",
-				}
-			}
-		end
-	end
-)
-

build/toolchain.py

@@ -0,0 +1,11 @@
+import platform
+
+_is_windows = (platform.system() == "Windows")
+
+class Toolchain:
+    PREFIX = ""
+    EXE = ".exe" if _is_windows else ""
+
+
+class HostToolchain(Toolchain):
+    PREFIX = "HOST"

build/utils.py

@@ -0,0 +1,105 @@
+from build.ab import (
+    Rule,
+    Target,
+    Targets,
+    filenameof,
+    filenamesof,
+    cwdStack,
+    error,
+    simplerule,
+    G
+)
+from os.path import relpath, splitext, join, basename, isfile
+from glob import iglob
+import fnmatch
+import subprocess
+import shutil
+
+
+def filenamesmatchingof(xs, pattern):
+    return fnmatch.filter(filenamesof(xs), pattern)
+
+
+def stripext(path):
+    return splitext(path)[0]
+
+
+def targetswithtraitsof(xs, trait):
+    return [t for t in xs if trait in t.traits]
+
+
+def collectattrs(*, targets, name, initial=[]):
+    s = set(initial)
+    for a in [t.args.get(name, []) for t in targets]:
+        s.update(a)
+    return sorted(s)
+
+
+def itemsof(pattern, root=None, cwd=None):
+    if not cwd:
+        cwd = cwdStack[-1]
+    if not root:
+        root = "."
+
+    pattern = join(cwd, pattern)
+    root = join(cwd, root)
+
+    result = {}
+    for f in iglob(pattern, recursive=True):
+        try:
+            if isfile(f):
+                result[relpath(f, root)] = f
+        except ValueError:
+            error(f"file '{f}' is not in root '{root}'")
+    return result
+
+
+def does_command_exist(cmd):
+    basecmd = cmd.strip().split()[0]
+    return shutil.which(basecmd)
+
+
+def shell(cmd):
+    r = subprocess.check_output([G.SHELL, "-c", cmd])
+    return r.decode("utf-8").strip()
+
+
+@Rule
+def objectify(self, name, src: Target, symbol):
+    simplerule(
+        replaces=self,
+        ins=["build/_objectify.py", src],
+        outs=[f"={basename(filenameof(src))}.h"],
+        commands=["$(PYTHON) $[ins[0]] $[ins[1]] " + symbol + " > $[outs]"],
+        label="OBJECTIFY",
+    )
+
+
+@Rule
+def test(
+    self,
+    name,
+    command: Target = None,
+    commands=None,
+    ins: Targets = None,
+    deps: Targets = None,
+    label="TEST",
+):
+    if command:
+        simplerule(
+            replaces=self,
+            ins=[command],
+            outs=["=sentinel"],
+            commands=["$[ins[0]]", "touch $[outs[0]]"],
+            deps=deps,
+            label=label,
+        )
+    else:
+        simplerule(
+            replaces=self,
+            ins=ins,
+            outs=["=sentinel"],
+            commands=commands + ["touch $[outs[0]]"],
+            deps=deps,
+            label=label,
+        )

build/zip.py

@@ -0,0 +1,25 @@
+from build.ab import (
+    Rule,
+    simplerule,
+    TargetsMap,
+    filenameof,
+)
+
+
+@Rule
+def zip(self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP"):
+    cs = ["$(PYTHON) build/_zip.py -z $[outs]"]
+
+    ins = []
+    for k, v in items.items():
+        cs += [f"-f {k} {filenameof(v)}"]
+        ins += [v]
+
+    simplerule(
+        replaces=self,
+        ins=ins,
+        deps=["build/_zip.py"],
+        outs=[f"={self.localname}." + extension],
+        commands=[" ".join(cs)],
+        label=label,
+    )

config.py

@@ -0,0 +1,11 @@
+import platform
+import os
+
+if os.getenv("BUILDTYPE") == "windows":
+    windows = True
+    osx = False
+    unix = False
+else:
+    windows = False
+    osx = platform.system() == "Darwin"
+    unix = True

dep/adflib/adf_nativ.h

@@ -2,33 +2,34 @@
 #define ADF_NATIV_H
 
 #ifdef __cplusplus
-extern "C" {
+extern "C"
+{
 #endif
 
 #include "adf_str.h"
 
 #define NATIVE_FILE 8001
 
-struct nativeDevice
-{
-	FILE* fd;
-};
+    struct nativeDevice
+    {
+        FILE* fd;
+    };
 
-struct nativeFunctions
-{
-    /* called by adfMount() */
-    RETCODE (*adfInitDevice)(struct Device*, char*, BOOL);
-    /* called by adfReadBlock() */
-    RETCODE (*adfNativeReadSector)(struct Device*, int32_t, int, uint8_t*);
-    /* called by adfWriteBlock() */
-    RETCODE (*adfNativeWriteSector)(struct Device*, int32_t, int, uint8_t*);
-    /* called by adfMount() */
-    BOOL (*adfIsDevNative)(char*);
-    /* called by adfUnMount() */
-    RETCODE (*adfReleaseDevice)(struct Device*);
-};
+    struct nativeFunctions
+    {
+        /* called by adfMount() */
+        RETCODE (*adfInitDevice)(struct Device*, char*, BOOL);
+        /* called by adfReadBlock() */
+        RETCODE (*adfNativeReadSector)(struct Device*, int32_t, int, uint8_t*);
+        /* called by adfWriteBlock() */
+        RETCODE (*adfNativeWriteSector)(struct Device*, int32_t, int, uint8_t*);
+        /* called by adfMount() */
+        BOOL (*adfIsDevNative)(char*);
+        /* called by adfUnMount() */
+        RETCODE (*adfReleaseDevice)(struct Device*);
+    };
 
-extern void adfInitNativeFct();
+    extern void adfInitNativeFct();
 
 #ifdef __cplusplus
 }

dep/adflib/build.mk

@@ -1,22 +0,0 @@
-ADFLIB_SRCS = \
-	dep/adflib/src/adf_bitm.c \
-	dep/adflib/src/adf_cache.c \
-	dep/adflib/src/adf_dir.c \
-	dep/adflib/src/adf_disk.c \
-	dep/adflib/src/adf_dump.c \
-	dep/adflib/src/adf_env.c \
-	dep/adflib/src/adf_file.c \
-	dep/adflib/src/adf_hd.c \
-	dep/adflib/src/adf_link.c \
-	dep/adflib/src/adf_raw.c \
-	dep/adflib/src/adf_salv.c \
-	dep/adflib/src/adf_util.c \
-
-ADFLIB_OBJS = $(patsubst %.c, $(OBJDIR)/%.o, $(ADFLIB_SRCS))
-$(ADFLIB_OBJS): CFLAGS += -Idep/adflib/src -Idep/adflib
-ADFLIB_LIB = $(OBJDIR)/libadflib.a
-$(ADFLIB_LIB): $(ADFLIB_OBJS)
-ADFLIB_CFLAGS = -Idep/adflib/src
-ADFLIB_LDFLAGS = $(ADFLIB_LIB)
-OBJS += $(ADFLIB_OBJS)
-

dep/adflib/build.py

@@ -0,0 +1,47 @@
+from build.c import clibrary
+
+clibrary(
+    name="adflib",
+    srcs=[
+        "./src/adf_bitm.c",
+        "./src/adf_bitm.h",
+        "./src/adf_cache.c",
+        "./src/adf_cache.h",
+        "./src/adf_dir.c",
+        "./src/adf_dir.h",
+        "./src/adf_disk.c",
+        "./src/adf_disk.h",
+        "./src/adf_dump.c",
+        "./src/adf_dump.h",
+        "./src/adf_env.c",
+        "./src/adf_env.h",
+        "./src/adf_file.c",
+        "./src/adf_file.h",
+        "./src/adf_hd.c",
+        "./src/adf_hd.h",
+        "./src/adf_link.c",
+        "./src/adf_link.h",
+        "./src/adf_raw.c",
+        "./src/adf_raw.h",
+        "./src/adf_salv.c",
+        "./src/adf_salv.h",
+        "./src/adf_str.h",
+        "./src/adf_util.c",
+        "./src/adf_util.h",
+        "./src/defendian.h",
+        "./src/hd_blk.h",
+        "./src/prefix.h",
+        "./adf_nativ.h",
+        "./config.h",
+        "./src/adflib.h",
+    ],
+    cflags=["-Wno-stringop-overflow"],
+    hdrs={
+        "adf_blk.h": "./src/adf_blk.h",
+        "adf_defs.h": "./src/adf_defs.h",
+        "adf_err.h": "./src/adf_err.h",
+        "adf_nativ.h": "./adf_nativ.h",
+        "adf_str.h": "./src/adf_str.h",
+        "adflib.h": "./src/adflib.h",
+    },
+)

dep/adflib/config.h

@@ -1,2 +1 @@
 /* empty config.h to keep the source happy */
-

dep/adflib/src/adf_bitm.h

@@ -27,25 +27,28 @@
  *
  */
 
-#include"adf_str.h"
-#include"prefix.h"
+#include "adf_str.h"
+#include "prefix.h"
 
 RETCODE adfReadBitmapBlock(struct Volume*, SECTNUM nSect, struct bBitmapBlock*);
-RETCODE adfWriteBitmapBlock(struct Volume*, SECTNUM nSect, struct bBitmapBlock*);
-RETCODE adfReadBitmapExtBlock(struct Volume*, SECTNUM nSect, struct bBitmapExtBlock*);
-RETCODE adfWriteBitmapExtBlock(struct Volume*, SECTNUM, struct bBitmapExtBlock* );
+RETCODE adfWriteBitmapBlock(
+    struct Volume*, SECTNUM nSect, struct bBitmapBlock*);
+RETCODE adfReadBitmapExtBlock(
+    struct Volume*, SECTNUM nSect, struct bBitmapExtBlock*);
+RETCODE adfWriteBitmapExtBlock(
+    struct Volume*, SECTNUM, struct bBitmapExtBlock*);
 
-SECTNUM adfGet1FreeBlock(struct Volume *vol);
-RETCODE adfUpdateBitmap(struct Volume *vol);
+SECTNUM adfGet1FreeBlock(struct Volume* vol);
+RETCODE adfUpdateBitmap(struct Volume* vol);
 PREFIX int32_t adfCountFreeBlocks(struct Volume* vol);
-RETCODE adfReadBitmap(struct Volume* , SECTNUM nBlock, struct bRootBlock* root);
+RETCODE adfReadBitmap(struct Volume*, SECTNUM nBlock, struct bRootBlock* root);
 BOOL adfIsBlockFree(struct Volume* vol, SECTNUM nSect);
 void adfSetBlockFree(struct Volume* vol, SECTNUM nSect);
 void adfSetBlockUsed(struct Volume* vol, SECTNUM nSect);
 BOOL adfGetFreeBlocks(struct Volume* vol, int nbSect, SECTNUM* sectList);
-RETCODE adfCreateBitmap(struct Volume *vol);
-RETCODE adfWriteNewBitmap(struct Volume *vol);
-void adfFreeBitmap(struct Volume *vol);
+RETCODE adfCreateBitmap(struct Volume* vol);
+RETCODE adfWriteNewBitmap(struct Volume* vol);
+void adfFreeBitmap(struct Volume* vol);
 
 #endif /* ADF_BITM_H */