Try uploading build artifacts.

Use mingw32 to build the Windows client, removing the cygwin dependency.

.appveyor.yml

@@ -0,0 +1,33 @@
+version: '{branch}.{build}'
+clone_depth: 1
+
+environment:
+  MSYSTEM: MINGW32
+
+init:
+  - git config --global core.autocrlf input
+
+install:
+  - set PATH=c:\msys64\mingw32\bin;c:\msys64\usr\bin;c:\msys64\bin;%PATH%
+  - echo %PATH%
+  - pacman -S --noconfirm --needed make ninja mingw-w64-i686-libusb mingw-w64-i686-sqlite3 mingw-w64-i686-zlib mingw-w64-i686-gcc
+
+build_script:
+  - make
+
+artifacts:
+  - path: fluxengine.exe
+    name: fluxengine.exe
+
+deploy:
+  release: fluxengine-windows-client-v$(appveyor_build_version)
+  description: FluxEngine Windows client
+  provider: GitHub
+  auth_token:
+    secure: dfJjj7fWCoDUz+Ni11OcNPB/U3TNJFwNA2AsL++ChFjniUsZLlC6SDWHiL/t4FZo
+  artifact: fluxengine.exe
+  draft: false
+  prerelease: false
+  on:
+    branch: master
+

.hgignore

@@ -3,4 +3,6 @@ streams
 .*\.flux
 .*\.img
 .*\.raw
+.vscode
+remote
 

.travis.yml

@@ -1,4 +1,6 @@
-language: generic
+language: shell
+git:
+    depth: 1
 
 matrix:
     include:
@@ -8,9 +10,11 @@ matrix:
             dist: xenial
             compiler: gcc
             env: CXX=g++-8
+            script:
+              - make
         -
             os: osx
-            env: HOMEBREW_NO_AUTO_UPDATE=1
+            osx_image: xcode10.2
             compiler: clang
 
 addons:
@@ -20,18 +24,14 @@ addons:
         - ubuntu-toolchain-r-test
         packages:
         - ninja-build
-        - meson
         - libusb-1.0-0-dev
         - libsqlite3-dev
         - g++-8
     homebrew:
         packages:
         - ninja
-        - meson
-
-git:
-    depth: 1
 
 script:
-    - make
+- make
+
 

FluxEngine.cydsn/FluxEngine.cydwr

@@ -530,7 +530,7 @@
     <Group key="v1">
       <Data key="cy_boot" value="cy_boot_v5_80" />
       <Data key="Em_EEPROM_Dynamic" value="Em_EEPROM_Dynamic_v2_20" />
-      <Data key="LIN_Dynamic" value="LIN_Dynamic_v4_0" />
+      <Data key="LIN_Dynamic" value="LIN_Dynamic_v5_0" />
     </Group>
   </Group>
   <Data key="DataVersionKey" value="2" />
@@ -571,6 +571,7 @@
       <Data key="e51063a9-4fad-40c7-a06b-7cc4b137dc18" value="DSKCHG" />
       <Data key="ea7ee228-8b3f-426c-8bb8-cd7a81937769" value="DIR" />
       <Data key="ed092b9b-d398-4703-be89-cebf998501f6" value="UartTx" />
+      <Data key="fbd1f839-40f9-498e-a48b-5f3048ea5c3d/52f31aa9-2f0a-497d-9a1f-1424095e13e6" value="SW_Tx_UART_1_tx" />
       <Data key="fede1767-f3fd-4021-b3d7-8f9d88f36f9b" value="DRVSA" />
       <Data key="fff78075-035e-43d7-8577-bc5be4d21926" value="WGATE" />
     </Group>
@@ -3809,6 +3810,11 @@
         <Data key="Port Format" value="12,7" />
       </Group>
     </Group>
+    <Group key="fbd1f839-40f9-498e-a48b-5f3048ea5c3d/52f31aa9-2f0a-497d-9a1f-1424095e13e6">
+      <Group key="0">
+        <Data key="Port Format" value="2,0" />
+      </Group>
+    </Group>
     <Group key="fede1767-f3fd-4021-b3d7-8f9d88f36f9b">
       <Group key="0">
         <Data key="Port Format" value="12,2" />

FluxEngine.cydsn/FluxEngine.cyprj

@@ -39,6 +39,20 @@
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="crunch.c" persistent="..\lib\common\crunch.c">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_C;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="crunch.h" persistent="..\lib\common\crunch.h">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="HEADER;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 </dependencies>
 </CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
@@ -1692,20 +1706,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART" persistent="">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART.c" persistent="Generated_Source\PSoC5\UART.c">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART.h" persistent="Generated_Source\PSoC5\UART.h">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -1731,6 +1745,34 @@
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmGnu.s" persistent="Generated_Source\PSoC5\UART_AsmGnu.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;b98f980c-3bd1-4fc7-a887-c56a20a46fdd;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmRv.s" persistent="Generated_Source\PSoC5\UART_AsmRv.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;fdb8e1ae-f83a-46cf-9446-1d703716f38a;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_PVT.h" persistent="Generated_Source\PSoC5\UART_PVT.h">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="HEADER;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmIar.s" persistent="Generated_Source\PSoC5\UART_AsmIar.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;e9305a93-d091-4da5-bdc7-2813049dcdbf;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 </dependencies>
 </CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
@@ -2164,20 +2206,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep4_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep4_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2190,20 +2232,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep3_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep3_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2216,20 +2258,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep2_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep2_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2242,20 +2284,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep1_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep1_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -3428,7 +3470,7 @@
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Optimization Level" v="" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Link Time Optimization" v="" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Fat LTO objects" v="" />
-<name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Pre Build Commands" v="" />
+<name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Pre Build Commands" v="cscript patcher.vbs" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Post Build Commands" v="" />
 </name>
 </platform>
@@ -3594,6 +3636,14 @@
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@Linker@Command Line@Command Line" v="--semihosting" />
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@User Commands@General@Pre Build Commands" v="" />
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@User Commands@General@Post Build Commands" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@General@Output Directory" v="${ProjectDir}\${ProcessorType}\${Platform}\${Config}" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Assembly@Command Line@Command Line" v="-s+ -M&lt;&gt; -w+ -r -DNDEBUG --fpu None" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@C/C++@General@Preprocessor Definitions" v="-D NDEBUG -D CY_CORE_ID=0" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@C/C++@Command Line@Command Line" v="-D NDEBUG -D CY_CORE_ID=0 --debug --endian=little -e --fpu=None --no_wrap_diagnostics" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Library Generation@Command Line@Command Line" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Linker@Command Line@Command Line" v="--semihosting" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@User Commands@General@Pre Build Commands" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@User Commands@General@Post Build Commands" v="" />
 </name>
 </platform>
 </platforms>

FluxEngine.cydsn/main.c

@@ -1,13 +1,15 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdarg.h>
 #include <setjmp.h>
 #include "project.h"
 #include "../protocol.h"
+#include "../lib/common/crunch.h"
 
 #define MOTOR_ON_TIME 5000 /* milliseconds */
 #define STEP_INTERVAL_TIME 6 /* ms */
-#define STEP_SETTLING_TIME 40 /* ms */
+#define STEP_SETTLING_TIME 50 /* ms */
 
 #define DISKSTATUS_WPT    1
 #define DISKSTATUS_DSKCHG 2
@@ -22,12 +24,13 @@ static bool motor_on = false;
 static uint32_t motor_on_time = 0;
 static bool homed = false;
 static int current_track = 0;
-static int current_drive = 0;
+static uint8_t current_drive_flags = 0;
 
 #define BUFFER_COUNT 16
 #define BUFFER_SIZE 64
 static uint8_t td[BUFFER_COUNT];
 static uint8_t dma_buffer[BUFFER_COUNT][BUFFER_SIZE] __attribute__((aligned()));
+static uint8_t usb_buffer[BUFFER_SIZE] __attribute__((aligned()));
 static uint8_t dma_channel;
 #define NEXT_BUFFER(b) (((b)+1) % BUFFER_COUNT)
 
@@ -69,16 +72,17 @@ CY_ISR(replay_dma_finished_irq_cb)
         dma_underrun = true;
 }
 
-static void print(const char* s)
+static void print(const char* msg, ...)
 {
-    /* UART_PutString(s); */
-}
-
-static void printi(int i)
-{
-    char buffer[16];
-    sprintf(buffer, "%d", i);
-    print(buffer);
+    char buffer[64];
+    
+    va_list ap;
+    va_start(ap, msg);
+    vsnprintf(buffer, sizeof(buffer), msg, ap);
+    va_end(ap);
+    
+    UART_PutString(buffer);
+    UART_PutCRLF();
 }
 
 static void start_motor(void)
@@ -89,9 +93,6 @@ static void start_motor(void)
         CyDelay(1000);
         homed = false;
     }
-    
-    if (DISKSTATUS_REG_Read() & DISKSTATUS_DSKCHG)
-        homed = false;
 
     motor_on_time = clock;
     motor_on = true;
@@ -106,6 +107,7 @@ static void wait_until_writeable(int ep)
 
 static void send_reply(struct any_frame* f)
 {
+    print("reply 0x%02x", f->f.type);
     wait_until_writeable(FLUXENGINE_CMD_IN_EP_NUM);
     USBFS_LoadInEP(FLUXENGINE_CMD_IN_EP_NUM, (uint8_t*) f, f->f.size);
 }
@@ -136,8 +138,10 @@ static void cmd_get_version(struct any_frame* f)
 
 static void step(int dir)
 {
+    STEP_REG_Write(dir);
+    CyDelayUs(1);
     STEP_REG_Write(dir | 2);
-    CyDelay(1);
+    CyDelayUs(1);
     STEP_REG_Write(dir);
     CyDelay(STEP_INTERVAL_TIME);
 }
@@ -147,6 +151,7 @@ static void seek_to(int track)
     start_motor();
     if (!homed)
     {
+        print("homing");
         while (!TRACK0_REG_Read())
             step(STEP_TOWARDS0);
             
@@ -155,11 +160,19 @@ static void seek_to(int track)
         
         homed = true;
         current_track = 0;
-        CyDelay(1); /* for direction change */
+        CyDelayUs(1); /* for direction change */
     }
     
+    print("beginning seek from %d to %d", current_track, track);
     while (track != current_track)
     {
+        if (TRACK0_REG_Read())
+        {
+            if (current_track != 0)
+                print("unexpectedly detected track 0");
+            current_track = 0;
+        }
+        
         if (track > current_track)
         {
             step(STEP_AWAYFROM0);
@@ -170,8 +183,10 @@ static void seek_to(int track)
             step(STEP_TOWARDS0);
             current_track--;
         }
+        CyWdtClear();
     }
     CyDelay(STEP_SETTLING_TIME);
+    print("finished seek");
 }
 
 static void cmd_seek(struct seek_frame* f)
@@ -276,11 +291,16 @@ static void cmd_read(struct read_frame* f)
 
     /* Wait for the beginning of a rotation. */
         
+    print("wait");
     index_irq = false;
     while (!index_irq)
         ;
     index_irq = false;
     
+    crunch_state_t cs = {};
+    cs.outputptr = usb_buffer;
+    cs.outputlen = BUFFER_SIZE;
+    
     dma_writing_to_td = 0;
     dma_reading_from_td = -1;
     dma_underrun = false;
@@ -302,6 +322,8 @@ static void cmd_read(struct read_frame* f)
     int revolutions = f->revolutions;
     while (!dma_underrun)
     {
+        CyWdtClear();
+
         /* Have we reached the index pulse? */
         if (index_irq)
         {
@@ -319,30 +341,47 @@ static void cmd_read(struct read_frame* f)
                 goto abort;
         }
 
-        while (USBFS_GetEPState(FLUXENGINE_DATA_IN_EP_NUM) != USBFS_IN_BUFFER_EMPTY)
+        uint8_t dma_buffer_usage = 0;
+        while (dma_buffer_usage < BUFFER_SIZE)
         {
-            if (index_irq || dma_underrun)
-                goto abort;
-        }
+            cs.inputptr = dma_buffer[dma_reading_from_td] + dma_buffer_usage;
+            cs.inputlen = BUFFER_SIZE - dma_buffer_usage;
+            crunch(&cs);
+            dma_buffer_usage += BUFFER_SIZE - cs.inputlen;
+            count++;
+            if (cs.outputlen == 0)
+            {
+                while (USBFS_GetEPState(FLUXENGINE_DATA_IN_EP_NUM) != USBFS_IN_BUFFER_EMPTY)
+                {
+                    if (index_irq || dma_underrun)
+                        goto abort;
+                }
 
-        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, dma_buffer[dma_reading_from_td], BUFFER_SIZE);
+                USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, usb_buffer, BUFFER_SIZE);
+                cs.outputptr = usb_buffer;
+                cs.outputlen = BUFFER_SIZE;
+            }
+        }
         dma_reading_from_td = NEXT_BUFFER(dma_reading_from_td);
-        count++;
     }
 abort:
     CyDmaChSetRequest(dma_channel, CY_DMA_CPU_TERM_CHAIN);
     while (CyDmaChGetRequest(dma_channel))
         ;
 
+    donecrunch(&cs);
+    wait_until_writeable(FLUXENGINE_DATA_IN_EP_NUM);
+    unsigned zz = cs.outputlen;
+    if (cs.outputlen != BUFFER_SIZE)
+        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, usb_buffer, BUFFER_SIZE-cs.outputlen);
+    if ((cs.outputlen == BUFFER_SIZE) || (cs.outputlen == 0))
+        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, NULL, 0);
     wait_until_writeable(FLUXENGINE_DATA_IN_EP_NUM);
-    USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, NULL, 0);
     deinit_dma();
 
     if (dma_underrun)
     {
-        print("underrun after ");
-        printi(count);
-        print(" packets\r");
+        print("underrun after %d packets");
         send_error(F_ERROR_UNDERRUN);
     }
     else
@@ -350,6 +389,7 @@ abort:
         DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_READ_REPLY);
         send_reply(&r);
     }
+    print("count=%d i=%d d=%d zz=%d", count, index_irq, dma_underrun, zz);
 }
 
 static void init_replay_dma(void)
@@ -395,32 +435,80 @@ static void cmd_write(struct write_frame* f)
 
     init_replay_dma();
     bool writing = false; /* to the disk */
-    bool listening = false;
     bool finished = false;
     int packets = f->bytes_to_write / FRAME_SIZE;
-    int count_read = 0;
     int count_written = 0;
+    int count_read = 0;
     dma_writing_to_td = 0;
     dma_reading_from_td = -1;
     dma_underrun = false;
 
+    crunch_state_t cs = {};
+    cs.outputlen = BUFFER_SIZE;
+    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
+    
     int old_reading_from_td = -1;
     for (;;)
     {
-        if (dma_reading_from_td != old_reading_from_td)
-        {
-            count_written++;
-            old_reading_from_td = dma_reading_from_td;
-        }
+        /* Read data from USB into the buffers. */
         
-        if (dma_reading_from_td != -1)
+        if (NEXT_BUFFER(dma_writing_to_td) != dma_reading_from_td)
         {
-            /* We want to be writing to disk. */
+            if (writing && (dma_underrun || index_irq))
+                goto abort;
             
-            if (!writing)
+            /* Read crunched data, if necessary. */
+            
+            if (cs.inputlen == 0)
             {
-                print("start writing\r");
+                if (finished)
+                {
+                    /* There's no more data to read, so fake some. */
+                    
+                    for (int i=0; i<BUFFER_SIZE; i++)
+                        usb_buffer[i+0] = 0x7f;
+                    cs.inputptr = usb_buffer;
+                    cs.inputlen = BUFFER_SIZE;
+                }
+                else
+                {
+                    while (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) != USBFS_OUT_BUFFER_FULL)
+                    {
+                        if (writing && (dma_underrun || index_irq))
+                            goto abort;
+                    }
 
+                    int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, usb_buffer);
+                    cs.inputptr = usb_buffer;
+                    cs.inputlen = length;
+                    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
+
+                    count_read++;
+                    if ((length < FRAME_SIZE) || (count_read == packets))
+                        finished = true;
+                }
+            }
+            
+            /* If there *is* data waiting in the buffer, uncrunch it. */
+            
+            if (cs.inputlen != 0)
+            {
+                cs.outputptr = dma_buffer[dma_writing_to_td] + BUFFER_SIZE - cs.outputlen;
+                uncrunch(&cs);
+                if (cs.outputlen == 0)
+                {
+                    /* Completed a DMA buffer; queue it for writing. */
+                    
+                    dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
+                    cs.outputlen = BUFFER_SIZE;
+                }
+            }
+            
+            /* If we have a full buffer, start writing. */
+            if ((dma_reading_from_td == -1) && (dma_writing_to_td == BUFFER_COUNT-1))
+            {
+                dma_reading_from_td = old_reading_from_td = 0;
+                
                 /* Start the DMA engine. */
                 
                 SEQUENCER_DMA_FINISHED_IRQ_Enable();
@@ -441,90 +529,38 @@ static void cmd_write(struct write_frame* f)
                 ERASE_REG_Write(1); /* start erasing! */
                 SEQUENCER_CONTROL_Write(0); /* start writing! */
             }
-            
-            /* ...unless we reach the end of the track or suffer underrung, of course. */
-            
-            if (index_irq || dma_underrun)
-                break;
         }
         
-        if (NEXT_BUFFER(dma_writing_to_td) != dma_reading_from_td)
+        if (writing && (dma_underrun || index_irq))
+            goto abort;
+
+        if (dma_reading_from_td != old_reading_from_td)
         {
-            /* We're ready for more data. */
-            
-            if (finished)
-            {
-                /* The USB stream has stopped early, so just fake data to keep the writer happy. */
-                
-                for (int i=0; i<BUFFER_SIZE; i++)
-                    dma_buffer[dma_writing_to_td][i] = 0x80;
-                dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
-            }
-            else
-            {
-                /* Make sure we're waiting for USB data. */
-                
-                if (!listening)
-                {
-                    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
-                    listening = true;
-                }
-                         
-                /* Is more data actually ready? */
-                
-                if (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) == USBFS_OUT_BUFFER_FULL)
-                {            
-                    int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, dma_buffer[dma_writing_to_td]);
-                    listening = false;
-                    dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
-                    
-                    count_read++;
-                    if ((length < FRAME_SIZE) || (count_read == packets))
-                        finished = true;
-                }
-            }
-            
-            /* Only start writing once the buffer is full. */
-            
-            if ((dma_reading_from_td == -1) && (dma_writing_to_td == BUFFER_COUNT-1))
-                dma_reading_from_td = old_reading_from_td = 0;
+            count_written++;
+            old_reading_from_td = dma_reading_from_td;
         }
     }
+abort:
     SEQUENCER_DMA_FINISHED_IRQ_Disable();
 
     SEQUENCER_CONTROL_Write(1); /* reset */
     if (writing)
     {
         ERASE_REG_Write(0);
-        print("stop writing after ");
-        printi(count_written);
-        print("\r");
         CyDmaChSetRequest(dma_channel, CY_DMA_CPU_TERM_CHAIN);
         while (CyDmaChGetRequest(dma_channel))
             ;
         CyDmaChDisable(dma_channel);
     }
     
-    if (dma_underrun)
-    {
-        print("underrun after ");
-        printi(count_read);
-        print(" out of ");
-        printi(packets);
-        print(" packets read\r");
-    }
-
-    DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_WRITE_REPLY);
-
+    //debug("p=%d cr=%d cw=%d f=%d l=%d w=%d index=%d underrun=%d", packets, count_read, count_written, finished, listening, writing, index_irq, dma_underrun);
     if (!finished)
     {
-        if (!listening)
-            USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
         while (count_read < packets)
         {
             if (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) == USBFS_OUT_BUFFER_FULL)
             {
-                int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, dma_buffer[0]);
+                int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, usb_buffer);
                 if (length < FRAME_SIZE)
                     break;
                 USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
@@ -542,6 +578,7 @@ static void cmd_write(struct write_frame* f)
         return;
     }
 
+    DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_WRITE_REPLY);
     send_reply((struct any_frame*) &r);
 }
 
@@ -551,7 +588,7 @@ static void cmd_erase(struct erase_frame* f)
     seek_to(current_track);    
     /* Disk is now spinning. */
     
-    print("start erasing\r");
+    print("start erasing");
     index_irq = false;
     while (!index_irq)
         ;
@@ -560,7 +597,7 @@ static void cmd_erase(struct erase_frame* f)
     while (!index_irq)
         ;
     ERASE_REG_Write(0);
-    print("stop erasing\r");
+    print("stop erasing");
 
     DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_ERASE_REPLY);
     send_reply((struct any_frame*) &r);
@@ -568,10 +605,10 @@ static void cmd_erase(struct erase_frame* f)
 
 static void cmd_set_drive(struct set_drive_frame* f)
 {
-    if (current_drive != f->drive)
+    if (current_drive_flags != f->drive_flags)
     {
-        current_drive = f->drive;
-        DRIVE_REG_Write(current_drive);
+        current_drive_flags = f->drive_flags;
+        DRIVE_REG_Write(current_drive_flags);
         homed = false;
     }
     
@@ -585,6 +622,7 @@ static void handle_command(void)
     (void) usb_read(FLUXENGINE_CMD_OUT_EP_NUM, input_buffer);
 
     struct any_frame* f = (struct any_frame*) input_buffer;
+    print("command 0x%02x", f->f.type);
     switch (f->f.type)
     {
         case F_FRAME_GET_VERSION_CMD:
@@ -637,7 +675,7 @@ int main(void)
     CAPTURE_DMA_FINISHED_IRQ_StartEx(&capture_dma_finished_irq_cb);
     SEQUENCER_DMA_FINISHED_IRQ_StartEx(&replay_dma_finished_irq_cb);
     DRIVE_REG_Write(0);
-    /* UART_Start(); */
+    UART_Start();
     USBFS_Start(0, USBFS_DWR_VDDD_OPERATION);
     
     CyWdtStart(CYWDT_1024_TICKS, CYWDT_LPMODE_DISABLED);
@@ -660,10 +698,10 @@ int main(void)
         
         if (!USBFS_GetConfiguration() || USBFS_IsConfigurationChanged())
         {
-            print("Waiting for USB...\r");
+            print("Waiting for USB...");
             while (!USBFS_GetConfiguration())
                 ;
-            print("USB ready\r");
+            print("USB ready");
             USBFS_EnableOutEP(FLUXENGINE_CMD_OUT_EP_NUM);
         }
         
@@ -671,7 +709,7 @@ int main(void)
         {
             handle_command();
             USBFS_EnableOutEP(FLUXENGINE_CMD_OUT_EP_NUM);
-            print("idle\r");
+            print("idle");
         }
     }
 }

Makefile

@@ -1,6 +1,36 @@
-all: .obj/build.ninja
-	@ninja -C .obj test
-
-.obj/build.ninja:
-	@mkdir -p .obj
-	meson .obj
+PACKAGES = zlib sqlite3 libusb-1.0
+
+ifeq ($(OS), Windows_NT)
+export CXX = /mingw32/bin/g++
+export AR = /mingw32/bin/ar rcs
+export STRIP = /mingw32/bin/strip
+export CFLAGS = -O3 -g --std=c++14 -I/mingw32/include/libusb-1.0
+export LDFLAGS = -O3
+export LIBS = -static -lz -lsqlite3 -lusb-1.0
+export EXTENSION = .exe
+else
+export CXX = g++
+export AR = ar rcs
+export STRIP = strip
+export CFLAGS = -Og -g --std=c++14 $(shell pkg-config --cflags $(PACKAGES))
+export LDFLAGS = -Og
+export LIBS = $(shell pkg-config --libs $(PACKAGES))
+export EXTENSION =
+endif
+
+CFLAGS += -Ilib -Idep/fmt
+
+export OBJDIR = .obj
+
+all: .obj/build.ninja
+	@ninja -f .obj/build.ninja -v
+
+clean:
+	@echo CLEAN
+	@rm -rf $(OBJDIR)
+
+.obj/build.ninja: mkninja.sh Makefile
+	@echo MKNINJA $@
+	@mkdir -p $(OBJDIR)
+	@sh $< > $@
+

README.md

@@ -29,6 +29,13 @@ the board. Sorry for the inconvenience, but it means you don't have to modify
 the board any more to make it work. If you built the hardware prior to then,
 you'll need to adjust it.
 
+**Another important note.** On 2019-07-03 I've revamped the build process and
+the (command line) user interface. It should be much nicer now, not least in
+that there's a single client binary with all the functionality in it. The
+interface is a little different, but not much. The build process is now
+better (simpler). See [the building](doc/building.md) and
+[using](doc/using.md) pages for more details.
+
 Where?
 ------
 
@@ -52,9 +59,8 @@ following friendly articles:
   - [Using a FluxEngine](doc/using.md) ∾ what to do with your new hardware ∾
     flux files and image files ∾ knowing what you're doing
 
-  - [Reading dubious disks](doc/problems.md) ∾ it's not an exact science ∾
-    the sector map ∾ clock detection and the histogram ∾ tuning the clock ∾
-    manual adjustment
+  - [Troubleshooting dubious disks](doc/problems.md) ∾ it's not an exact science ∾
+    the sector map ∾ clock detection and the histogram
 
 Which?
 ------
@@ -66,7 +72,8 @@ decoder based on Kryoflux (or other) dumps I've found. I don't (yet) have
 real, physical disks in my hand to test the capture process.
 
 Unicorns (🦄) are completely real --- this means that I've read actual,
-physical disks with these formats and so know they work.
+physical disks with these formats and so know they work (or had reports from
+people who've had it work).
 
 ### Old disk formats
 
@@ -76,11 +83,12 @@ physical disks with these formats and so know they work.
 | [Acorn ADFS](doc/disk-acornadfs.md)      |  🦄   |        | single- and double- sided           |
 | [Acorn DFS](doc/disk-acorndfs.md)        |  🦄   |        |                                     |
 | [Ampro Little Board](doc/disk-ampro.md)  |  🦖   |        |                                     |
-| [Apple II DOS 3.3](doc/disk-apple2.md)   |  🦖   |        | doesn't do logical sector remapping |
+| [Apple II DOS 3.3](doc/disk-apple2.md)   |  🦄   |        | doesn't do logical sector remapping |
 | [Amiga](doc/disk-amiga.md)               |  🦄   |        |                                     |
 | [Commodore 64 1541](doc/disk-c64.md)     |  🦖   |        | and probably the other GCR formats  |
 | [Brother 120kB](doc/disk-brother.md)     |  🦄   |        |                                     |
 | [Brother 240kB](doc/disk-brother.md)     |  🦄   |   🦄   |                                     |
+| [Brother FB-100](doc/disk-fb100.md)      |  🦖   |        | Tandy Model 100, Husky Hunter, knitting machines |
 | [Macintosh 800kB](doc/disk-macintosh.md) |  🦖   |        | and probably the 400kB too          |
 | [TRS-80](doc/disk-trs80.md)              |  🦖   |        | a minor variation of the IBM scheme |
 {: .datatable }
@@ -100,6 +108,7 @@ at least, check the CRC so what data's there is probably good.
 | [Victor 9000](doc/disk-victor9k.md)      |  🦖   |        | 8-inch        |
 | [Zilog MCZ](doc/disk-zilogmcz.md)        |  🦖   |        | 8-inch _and_ hard sectors |
 {: .datatable }
+
 ### Notes
 
   - IBM PC disks are the lowest-common-denominator standard. A number of other

dep/fmt/fmt/format.h

@@ -3416,7 +3416,7 @@ auto join(const Range &range, wstring_view sep)
 
   **Example**::
 
-    #include <fmt/format.h>
+    #include "fmt/format.h"
 
     std::string answer = fmt::to_string(42);
   \endrst
@@ -3697,7 +3697,7 @@ FMT_END_NAMESPACE
   **Example**::
 
     #define FMT_STRING_ALIAS 1
-    #include <fmt/format.h>
+    #include "fmt/format.h"
     // A compile-time error because 'd' is an invalid specifier for strings.
     std::string s = format(fmt("{:d}"), "foo");
   \endrst

doc/building.md

@@ -141,16 +141,31 @@ the port and proceed normally.
 
 The client software is where the intelligence, such as it is, is. It's pretty
 generic libusb stuff and should build and run on Windows, Linux and OSX as
-well, although on Windows I've only ever used it with Cygwin. You'll need the
-`sqlite3`, `meson` and `ninja` packages (which should be easy to come by in
-your distribution). Just do `make` and it should build.
+well, although on Windows it'll need MSYS2 and mingw32. You'll need to
+install some support packages.
+
+  - For Linux (this is Ubuntu, but this should apply to Debian too):
+  `ninja-build`, `libusb-1.0-0-dev`, `libsqlite3-dev`.
+  - For OSX with Homebrew: `ninja`.
+  - For Windows with MSYS2: `make`, `ninja`, `mingw-w64-i686-libusb`,
+  `mingw-w64-i686-sqlite3`, `mingw-w64-i686-zlib`, `mingw-w64-i686-gcc`.
+
+These lists are not necessarily exhaustive --- plaese [get in
+touch](https://github.com/davidgiven/fluxengine/issues/new) if I've missed
+anything.
+
+All systems build by just doing `make`. You should end up with a single
+executable in the current directory, called `fluxengine`. It has minimal
+dependencies and you should be able to put it anywhere.
 
 If it doesn't build, please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new).
 
 ## Next steps
 
-The board's now assembled and programmed. Plug it into your drive, strip the plastic off the little USB connector and plug that into your computer, and you're ready to start using it.
+The board's now assembled and programmed. Plug it into your drive, strip the
+plastic off the little USB connector and plug that into your computer, and
+you're ready to start using it.
 
 I _do_ make updates to the firmware whenever necessary, so you may need to
 reprogram it at intervals; you may want to take this into account if you

doc/disk-acornadfs.md

@@ -31,7 +31,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readadfs
+fluxengine read adfs
 ```
 
 You should end up with an `adfs.img` of the appropriate size for your disk

doc/disk-acorndfs.md

@@ -19,7 +19,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readdfs
+fluxengine read dfs
 ```
 
 You should end up with an `dfs.img` of the appropriate size for your disk

doc/disk-aeslanier.md

@@ -3,7 +3,7 @@ Disk: AES Lanier word processor
 
 Back in 1980 Lanier released a series of very early integrated word processor
 appliances, the No Problem. These were actually [rebranded AES Data Superplus
-machines](http://vintagecomputers.site90.net/aes/). They wrer gigantic,
+machines](http://vintagecomputers.site90.net/aes/). They were gigantic,
 weighed 40kg, and one example I've found cost £13,000 in 1981 (the equivalent
 of nearly £50,000 in 2018!).
 
@@ -17,9 +17,10 @@ indicating to the hardware where the sectors start. The encoding scheme
 itself is [MMFM (aka
 M2FM)](http://www.retrotechnology.com/herbs_stuff/m2fm.html), an early
 attempt at double-density disk encoding which rapidly got obsoleted by the
-simpler MFM. Even aside from the encoding, the format on disk was strange;
-unified sector header/data records, so that the sector header (containing the
-sector and track number) is actually inside the user data.
+simpler MFM --- and the bytes are stored on disk _backwards_. Even aside from
+the encoding, the format on disk was strange; unified sector header/data
+records, so that the sector header (containing the sector and track number)
+is actually inside the user data.
 
 FluxEngine can read these, but I only have a single, fairly poor example of a
 disk image, and I've had to make a lot of guesses as to the sector format
@@ -32,10 +33,10 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readaeslanier
+fluxengine read aeslanier
 ```
 
 Useful references
 -----------------
 
-  * [SA800 Diskette Storage Drive - Theory Of Operations](http://www.hartetechnologies.com/manuals/Shugart/50664-1_SA800_TheorOp_May78.pdf): talks about MMFM a lot, but the Lanier machines didn't use this disk format.
+  * [SA800 Diskette Storage Drive - Theory Of Operations](http://www.hartetechnologies.com/manuals/Shugart/50664-1_SA800_TheorOp_May78.pdf): talks about MMFM a lot, but the Lanier machines didn't use this disk format.

doc/disk-amiga.md

@@ -16,7 +16,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readamiga
+fluxengine read amiga
 ```
 
 You should end up with an `amiga.adf` which is 901120 bytes long (for a

doc/disk-ampro.md

@@ -19,7 +19,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readampro
+fluxengine read ampro
 ```
 
 You should end up with an `ampro.img` which is 409600 or 819200 bytes long.

doc/disk-apple2.md

@@ -17,9 +17,9 @@ scheme applied to the data before it goes down on disk to speed up
 checksumming.
 
 Macintosh disks come in two varieties: the newer 1440kB ones, which are
-perfectly ordinary PC disks you should use `fe-readibm` to read, and the
-older 800kB disks (and 400kB for the single sides ones). They have 80 tracks
-and up to 12 sectors per track.
+perfectly ordinary PC disks you should use `fluxengine read ibm` to read, and
+the older 800kB disks (and 400kB for the single sides ones). They have 80
+tracks and up to 12 sectors per track.
 
 In addition, a lot of the behaviour of the drive was handled in software.
 This means that Apple II disks can do all kinds of weird things, including
@@ -42,7 +42,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readapple2
+fluxengine read apple2
 ```
 
 You should end up with an `apple2.img` which is 143360 bytes long.

doc/disk-brother.md

@@ -8,7 +8,7 @@ size.
 Different word processors use different disk formats --- the only ones
 supported by FluxEngine are the 120kB and 240kB 3.5" formats. The default
 options are for the 240kB format. For the 120kB format, which is 40 track, do
-`fe-readbrother -s :t=1-79x2`.
+`fluxengine read brother -s :t=1-79x2`.
 
 Apparently about 20% of Brother word processors have alignment issues which
 means that the disks can't be read by FluxEngine (because the tracks on the
@@ -36,7 +36,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readbrother
+fluxengine read brother
 ```
 
 You should end up with a `brother.img` which is 239616 bytes long.
@@ -47,7 +47,7 @@ Writing discs
 Just do:
 
 ```
-.obj/fe-writebrother
+fluxengine write brother
 ```
 
 ...and it'll write a `brother.img` file which is 239616 bytes long to the
@@ -108,9 +108,13 @@ To extract a file, do:
 Wildcards are supported, so use `'*'` for the filename (remember to quote it)
 if you want to extract everything.
 
-This is _extremely experimental_. The data structures I've figured out are
-mostly consistent, but it looks like there's always garbage in the last
-sector of each file, so maybe I'm not getting the file lengths right.
+The files are usually in the format known as WP-1, which aren't well
+supported by modern tools (to nobody's great surprise). Matthias Henckell has
+[reverse engineered the file
+format](https://mathesoft.eu/brother-wp-1-dokumente/) and has produced a
+(Windows-only, but runs in Wine) [tool which will convert these files into
+RTF](https://mathesoft.eu/sdm_downloads/wp2rtf/). This will only work on WP-1
+files.
 
 Any questions? please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new).
@@ -136,5 +140,6 @@ mcopy -i brother.img ::brother.doc linux.doc
 The word processor checks the media byte, unfortunately, so you'll need to
 change it back to 0x58 before writing an image to disk.
 
-Converting the equally proprietary file format to something readable is,
-unfortunately, out of scope for FluxEngine.
+The file format is not WP-1, and currently remains completely unknown,
+although it's probably related. If anyone knows anything about this, please
+[get in touch](https://github.com/davidgiven/fluxengine/issues/new).

doc/disk-c64.md

@@ -3,8 +3,8 @@ Disk: Commodore 64
 
 Commodore 64 disks come in two varieties: GCR, which are the overwhelming
 majority; and MFM, only used on the 1571 and 1581. The latter were (as far as
-I can tell) standard IBM PC format disks, so use `fe-readibm` to read them
-(and then [let me know if it
+I can tell) standard IBM PC format disks, so use `fluxengine read ibm` to
+read them (and then [let me know if it
 worked](https://github.com/davidgiven/fluxengine/issues/new).
 
 The GCR disks are much more interesting. They could store 170kB on a
@@ -31,7 +31,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readc64
+fluxengine read c64
 ```
 
 You should end up with an `c64.img` which is 187136 bytes long (for a normal

doc/disk-durangof85.md

@@ -15,7 +15,8 @@ it for the following photo...
 <img src="durangof85.jpg" style="max-width: 60%" alt="A Durango F85, held precariously">
 </div>
 
-...and even then, only for a few seconds.
+...and even then, they had to airbrush out the tendons in her neck from the
+effort!
 
 It used 5.25 soft-sectored disks storing an impressive-for-those-days
 480kBish on a side, using a proprietary 4-in-5 GCR encoding. They used 77
@@ -31,7 +32,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readf85
+fluxengine read f85
 ```
 
 You should end up with an `f85.img` which is 472064 bytes long.

doc/disk-fb100.md

@@ -0,0 +1,45 @@
+Disk: Brother FB-100
+====================
+
+The Brother FB-100 is a serial-attached smart floppy drive used by a several
+different machines for mass storage, including the Tandy Model 100 and
+clones, the Husky Hunter 2, and (bizarrely) several knitting machines. It was
+usually rebadged, sometimes with a cheap paper label stuck over the Brother
+logo, but the most common variant appears to be the Tandy Portable Disk Drive
+or TPDD:
+
+<div style="text-align: center">
+<a href="http://www.old-computers.com/museum/computer.asp?c=233&st=1"> <img src="tpdd.jpg" alt="A Tandy Portable Disk Drive"/></a>
+</div>
+
+It's a bit of an oddball: the disk encoding is FM with a very custom record
+scheme: 40-track single-sided 3.5" disks storing 100kB or so each. Each track
+had only _two_ sectors, each 1280 bytes, but with an additional 12 bytes of
+ID data used for filesystem management.
+
+There was also apparently a TPDD-2 which could store twice as much data, but
+I don't have access to one of those disks.
+
+Reading discs
+-------------
+
+Just do:
+
+```
+fluxengine read fb100
+```
+
+You should end up with an `fb100.img` of the appropriate size. It's a simple
+array of 80 1292-byte sectors (12 bytes for the ID record plus 1280 bytes for
+the data).
+
+References
+----------
+
+  - [Tandy Portable Disk Drive operations manual](http://www.classiccmp.org/cini/pdf/Tandy/Portable%20Disk%20Drive%20Operation%20Manual.pdf)
+
+  - [Tandy Portable Disk Drive service manual](https://archive.org/details/TandyPortableDiskDriveSoftwareManual26-3808s)
+
+  - [TPDD design notes (including a dump of the ROM)](http://bitchin100.com/wiki/index.php?title=TPDD_Design_Notes)
+
+  - [Knitting machine FB-100 resources](http://www.k2g2.org/wiki:brother_fb-100)

doc/disk-macintosh.md

@@ -2,9 +2,9 @@ Disk: Macintosh
 ===============
 
 Macintosh disks come in two varieties: the newer 1440kB ones, which are
-perfectly ordinary PC disks you should use `fe-readibm` to read, and the
-older 800kB disks (and 400kB for the single sides ones). They have 80 tracks
-and up to 12 sectors per track.
+perfectly ordinary PC disks you should use `fluxengine read ibm` to read, and
+the older 800kB disks (and 400kB for the single sides ones). They have 80
+tracks and up to 12 sectors per track.
 
 They are also completely insane.
 
@@ -37,7 +37,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readmac
+fluxengine read mac
 ```
 
 You should end up with an `mac.img` which is 1001888 bytes long (for a normal

doc/disk-trs80.md

@@ -22,7 +22,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readibm
+fluxengine read ibm
 ```
 
 You should end up with an `ibm.img` of the appropriate size. It's a simple

doc/disk-victor9k.md

@@ -1,22 +1,14 @@
 Disk: Victor 9000
 =================
 
-**Warning.** This is experimental; I haven't found a clean disk to read yet.
-The fragmented disk images which I have found ([from
-vintagecomputer.ca](http://vintagecomputer.ca/files/Victor%209000/)) get
-about 57% good sectors. It could just be that the disks are bad, but there
-could also be something wrong with my decode logic. If you have any Victor
-disks and want to give this a try for real, [please get in
-touch](https://github.com/davidgiven/fluxengine/issues/new).
-
 The Victor 9000 / Sirius One was a rather strange old 8086-based machine
 which used a disk format very reminiscent of the Commodore format; not a
 coincidence, as Chuck Peddle designed them both. They're 80-track, 512-byte
 sector GCR disks, with a variable-speed drive and a varying number of sectors
-per track --- from 19 to 12. Reports are that they're double-sided but I've
-never seen a double-sided disk image.
+per track --- from 19 to 12. Disks can be double-sided, meaning that they can
+store 1224kB per disk, which was almost unheard of back then.
 
-FluxEngine reads them (subject to the warning above).
+FluxEngine reads these.
 
 Reading discs
 -------------
@@ -24,7 +16,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readvictor9k
+fluxengine read victor9k
 ```
 
 You should end up with an `victor9k.img` which is 774656 bytes long.

doc/disk-zilogmcz.md

@@ -30,7 +30,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readzilogmcz
+fluxengine read zilogmcz
 ```
 
 You should end up with an `zilogmcz.img` which is 315392 bytes long.

doc/problems.md

@@ -1,64 +1,146 @@
 My disk won't read properly!
 ============================
 
-So you're trying to read a disk and it's not working. Well, you're not the only one. Floppy disks are unreliable and awkward things. FluxEngine can help, but the tools aren't very user friendly.
+So you're trying to read a disk and it's not working. Well, you're not the
+only one. Floppy disks are unreliable and awkward things. FluxEngine can
+help, but the tools aren't very user friendly.
+
+The good news is that as of 2019-04-30 the FluxEngine decoder algorithm has
+been written to be largely fire-and-forget and is mostly self-adjusting.
+However, there are still some things that can be tuned to produce better
+reads.
 
 The sector map
 --------------
 
-Every time I do a read, FluxEngine will give me a dump like this:
+Every time FluxEngine does a read, it produces a dump like this:
 
 ```
 H.SS Tracks --->
-0. 0 XBBXXXXXXXXXBXBBXBBXBBX..XXX.BXBBBXXX....BB...BB...........B.XXXX.X....BBBBBBXBB
-0. 1 X.BXXXXBBXXX.XBBXBXXBBB.XXXX.BBXBXXBX....BB...BX........BB.B.XXXX.X....BBBBBBXBX
-0. 2 X..XXXXXBXXX.XBBXXBXBBB.BXXB.BXBBBXXX...BXX.B.BB.........B.B.XXXX.X....BBBBBBXBX
-0. 3 X.BXXXXXXXXX.X.XXBBXBBX.XXXX.XXXXXXXX...BBX.X.XX......B....B.XXXB.X....BBBXBBXBX
-0. 4 X.BXXXXXXXXX.XBXXBXXBBB.XXXX.BXXXXXXX...BBB.X.BX......B....B.XXXX.X....BBBBBBXBX
-0. 5 X.BXXXXXXXXX.XBXXBBXBB.BXXXX.XXXXXXXX...XBB.X.XX.....B.....B.XXXX.B....BBBXBBXBX
-0. 6 X..XXXXBXXXX.X.XXBBXBBB.XXXB.XXBBBXXX..B.BB.B.XB.............XXXX.X....BBBXBBXBX
-0. 7 X..XXXXBXXXXBX.XXBBXBBB.BXXX.XXXBXXXX..BBXX.X.XX......B....B.XXXX.X....BBBXBBXBX
-0. 8 X.BXXXXBXXXXBX.XXBBXBBX.XXXX.XXXXXXXX..BXXX.X.XX......BB.B...XXXX.X....BBBBXBXBX
-0. 9 X.BXXXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX..XXXX.X.XX......X......XXXX.X....BBBBBBXBX
-0.10 X..XXXXBXXXXBX.XXBBXXB..BXXXBBXXBXXXX..BBXB.B.BB.............XXXX.X....BBBBBXXBX
-0.11 X..XXXXXXXXXBX.XXBBXBBB.XXXX.BXXXXXXX..XXXX.X.BB....B........XXXX.X....BBBBBBBBB
-0.12 X.BXXXXXXXXXBX.XXBXXXXB.XXXX.XXXXXXXX..XXXX.X.XX.........B.B.BXXX.B....XXXXXXXXX
-0.13 X..XBXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX..XXXX.X.XX....B.B.BB.BXXXXXXXXXXXXXXXXXXXX
-0.14 XB.XXXXXXXXXBX.XXBBXBBB.XXXB.XXBXXXXB..BBB..B.BBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.15 X..XXXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.16 X..XXBXXBXBXBBBXXBBXBBB.XXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.17 XBBXXXXB.XXXBXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.18 XBBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 369/1520 (24%)
-Missing sectors: 847/1520 (55%)
-Bad sectors: 304/1520 (20%)
+0. 0 XXXBXX...X.XXXX......?...........X...X.........X................................
+0. 1 ..X..X.X..XB.B.B........X...........X.......X...................................
+0. 2 X.XXXX.XX....XB.................X.X..X..X......X................................
+0. 3 X.X..XXXX..?XXXX..................XX.X..........................................
+0. 4 X.X..X....X.X.XX....?....?........XXXX..X.....X.................................
+0. 5 XXXX...?..X.XBX...?......C......C?.X.X...?....X..........X......................
+0. 6 XXXB.XX.XX???XXX...............CX.XXXX........X.................................
+0. 7 XX..XX.XC..?.X......B.......X...X..XX...C.......................................
+0. 8 X?.B...XXX.?..XX........X........XCXXX..X..X..X.................................
+0. 9 BB.XX.X.X.X...BX.........C.......XXX...........X.....X..........................
+0.10 BX.XX.XX.X..XX.B...X.............XXX........................................C...
+0.11 .C.X.C..BXXBXBX?X................XX..X......X...................................
+0.12 BX.XXX....BX..X......C....X......XXX.......XX..........................XXXXXXXXX
+0.13 X..BXX..X?.XX.X....X..............XXXX.X....X...............XXXXXXXXXXXXXXXXXXXX
+0.14 X...XXB..X.X..X....X...X..C........X?...........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.15 X.BX.XX.X.XXX.X...........X.....X..X..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.16 XBXX...XX.X.X.XX........B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.17 XXB..X.B....XX..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.18 XXCXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+Good sectors: 978/1520 (64%)
+Missing sectors: 503/1520 (33%)
+Bad sectors: 39/1520 (2%)
 80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
 ```
 
 This is the **sector map**, and is showing me the status of every sector it
 found on the disk. (Tracks on the X-axis, sectors on the Y-axis.) This is a
-very bad read from a [Victor 9000](victor9k.md) disk; good reads shouldn't
-look like this. A dot represents a good sector. A B is one where the CRC
-check failed; an X is one which couldn't be found at all.
+very bad read from a [Victor 9000](disk-victor9k.md) disk; good reads
+shouldn't look like this. A dot represents a good sector. A B is one where
+the CRC check failed; an X is one which couldn't be found at all; a ? is a
+sector which was found but contained no data.
 
-At the very bottom there's a summary: 24% good sectors. Let me try and improve
+At the very bottom there's a summary: 64% good sectors. Let me try and improve
 that.
 
 (You may notice the wedge of Xs in the bottom right. This is because the
 Victor 9000 uses a varying number of sectors per track; the short tracks in
 the middle of the disk store less. So, it's perfectly normal that those
 sectors are missing. This will affect the 'good sectors' score, so it's
-normal not to have 100% on this disk.)
+normal not to have 100% on this type of disk.)
+
+Clock errors
+------------
+
+When FluxEngine sees a track, it attempts to automatically guess the clock
+rate of the data in the track. It does this by looking for the magic bit
+pattern at the beginning of each sector record and measuring how long it
+takes. This is shown in the tracing FluxEngine produces as it runs. For
+example:
+
+```
+ 70.0: 868 ms in 427936 bytes
+       138 records, 69 sectors; 2.13us clock; 
+       logical track 70.0; 6656 bytes decoded.
+ 71.0: 890 ms in 387904 bytes
+       130 records, 65 sectors; 2.32us clock; 
+       logical track 71.0; 6144 bytes decoded.
+```
+
+Bits are then found by measuring the interval between pulses on the disk and
+comparing to this clock rate.
+
+However, floppy disk drives are extremely analogue devices and not necessarily calibrated very well, and the disk may be warped, or the rubber band which makes the drive work may have lost its bandiness, and so the bits are not necessarily precisely aligned. Because of this, FluxEngine can tolerate a certain amount of error. This is controlled by the `--bit-error-threshold` parameter. Varying this can have magical effects. For example, adding `--bit-error-threshold=0.4` turns the decode into this:
+
+```
+H.SS Tracks --->
+0. 0 ...B............................................................................
+0. 1 ...........B...B................................................................
+0. 2 ..............B.................................................................
+0. 3 ................................................................................
+0. 4 ................................................................................
+0. 5 ...B............................................................................
+0. 6 ...B.....B......................................................................
+0. 7 ...B...........B....B...........................................................
+0. 8 ................................................................................
+0. 9 .B............B.................................................................
+0.10 .B............BB................................................................
+0.11 B............B..................................................................
+0.12 ...B......B............................................................XXXXXXXXX
+0.13 ............B...............................................XXXXXXXXXXXXXXXXXXXX
+0.14 ...B......B.....................................XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.15 ..B.....BB..B.........................XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.16 ..B.....B.B.............B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.17 ..B....B........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.18 .B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+Good sectors: 1191/1520 (78%)
+Missing sectors: 296/1520 (19%)
+Bad sectors: 33/1520 (2%)
+80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
+```
+
+A drastic difference!
+
+The value of the parameter is the fraction of a clock of error to accept. The
+value typically varies from 0.0 to 0.5; the default is 0.2. Larger values
+make FluxEngine more tolerant, so trying 0.4 is the first thing to do when
+faced with a dubious disk. However, in some situations, increasing the value
+can actually _increase_ the error rate --- which is why 0.4 isn't the default
+--- so you'll need to experiment.
+
+That's the most common tuning parameter, but others are available:
+
+`--pulse-debounce-threshold` controls whether FluxEngine ignores pairs of pulses in rapid succession. This is common on some disks (I've observed them on Brother word processor disks).
+
+`--clock-interval-bias` adds a constant bias to the intervals between pulses
+before doing decodes. This is very occasionally necessary to get clean reads
+--- for example, if the machine which wrote the disk always writes pulses
+late. If you try this, use very small numbers (e.g. 0.02). Negative values
+are allowed.
+
+Both these parameters take a fraction of a clock as a parameter, and you'll
+probably never need to touch them.
 
 Clock detection
 ---------------
 
-When FluxEngine sees a track, it attempts to automatically guess the clock
-rate of the data in the track. It does this by computing a histogram of the
-spacing between pulses and attempting to detect the shortest peak. The
-histogram should look something like this.
+A very useful tool for examining problematic disks is `fluxengine inspect`.
+This will let you examine the raw flux on a disk (or flux file). It'll also
+guess the clock rate on the disk for you, using a simple statistical analysis
+of the pulse intervals on the disk. (Note that the tool only works on one
+track at a time.)
 
 ```
+$ fluxengine inspect -s good.flux:t=0:s=0
 Clock detection histogram:
 3.58    737 ▉
 3.67   3838 ████▊
@@ -96,6 +178,7 @@ Signal level: 3170
 Peak start:   42 (3.50 us)
 Peak end:     52 (4.33 us)
 Median:       47 (3.92 us)
+3.92us clock detected.
 ```
 
 That's _not_ the histogram from the Victor disk; that's an Apple II disk, and
@@ -109,190 +192,65 @@ So, what does my Victor 9000 histogram look like? Let's look at the
 histogram for a single track:
 
 ```
-$ fe-readvictor -s diskimage/:s=0:t=0 --show-clock-histogram
-Reading from: diskimage/:d=0:s=0:t=0
-  0.0: 829 ms in 316193 bytes
+$ fluxengine inspect -s dubious.flux:t=0:s=0
 Clock detection histogram:
-1.25    447 ▌
-1.33  16283 ████████████████████▎
-1.42  22879 ████████████████████████████▌
-1.50   4564 █████▋
-1.58   1272 █▌
-1.67  19594 ████████████████████████▍
-1.75  32059 ████████████████████████████████████████ 
-1.83  18042 ██████████████████████▌
-1.92   2249 ██▊
-2.00   8825 ███████████ 
-2.08  16031 ████████████████████ 
-2.17   5080 ██████▎
-2.25    409 ▌
-2.33   7216 █████████ 
-2.42   6269 ███████▊
-2.50    514 ▋
-2.58   5176 ██████▍
-2.67  13080 ████████████████▎
-2.75   6774 ████████▍
-2.83   1916 ██▍
-2.92  10880 █████████████▌
-3.00  21277 ██████████████████████████▌
-3.08  11625 ██████████████▌
-3.17   1028 █▎
+1.33   1904 █▉
+1.42  21669 ██████████████████████▌
+1.50   2440 ██▌
+1.58    469 ▍
+1.67   7261 ███████▌
+1.75   6808 ███████ 
+1.83   3088 ███▏
+1.92   2836 ██▉
+2.00   8897 █████████▎
+2.08   6200 ██████▍
 ...
-3.67   1910 ██▍
-3.75  19720 ████████████████████████▌
-3.83  12365 ███████████████▍
-3.92    814 █ 
-4.00   3144 ███▉
-4.08   5776 ███████▏
-4.17   3278 ████ 
-4.25   8487 ██████████▌
-4.33  15922 ███████████████████▊
-4.42   9656 ████████████ 
-4.50   1540 █▉
+2.25    531 ▌
+2.33   2802 ██▉
+2.42   2136 ██▏
+2.50   1886 █▉
+2.58  10110 ██████████▌
+2.67   8283 ████████▌
+2.75   7779 ████████ 
+2.83   2680 ██▊
+2.92  13908 ██████████████▍
+3.00  38431 ████████████████████████████████████████ 
+3.08  35708 █████████████████████████████████████▏
+3.17   5361 █████▌
 ...
-Noise floor:  320
-Signal level: 3205
-Peak start:   14 (1.17 us)
-Peak end:     39 (3.25 us)
-Median:       23 (1.92 us)
+3.75    294 ▎
+3.83    389 ▍
+3.92   1224 █▎
+4.00   3067 ███▏
+4.08   4092 ████▎
+4.17   6916 ███████▏
+4.25  25639 ██████████████████████████▋
+4.33  31407 ████████████████████████████████▋
+4.42  10209 ██████████▋
+4.50   1159 █▏
+...
+Noise floor:  384
+Signal level: 1921
+Peak start:   15 (1.25 us)
+Peak end:     26 (2.17 us)
+Median:       20 (1.67 us)
+1.67 us clock detected.
 ```
 
 That's... not good. The disk is very noisy, and the intervals between pulses
-are horribly distributed. The detected clock is 1.92us, which is clearly
-wrong.
+are horribly distributed. The detected clock from the decode is 1.45us, which
+does correspond more-or-less to a peak. You'll also notice that the
+double-clock interval is at 3.00us, which is _not_ twice 1.45us. The guessed
+clock by `fluxengine inspect` is 1.67us, which is clearly wrong.
 
-I can override the clock detection and specify the clock manually. 1.75us
-looks like a good candidate. Let's try that on track 0.
+This demonstrates that the statistical clock guessing isn't brilliant, which
+is why I've just rewritten the decoder not to use it; nevertheless, it's a
+useful tool for examining disks.
 
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0 --manual-clock-rate-us=1.75
-...skipped...
-No sectors in output; skipping analysis
-0 tracks, 0 heads, 0 sectors, 0 bytes per sector, 0 kB total
-```
+`fluxengine inspect` will also dump the raw flux data in various formats, but
+that's mostly only useful to me. Try `--dump-bits` to see the raw bit pattern
+on the disk (using the guessed clock, or `--manual-clock-rate-us` to set it
+yourself); `--dump-flux` will show you discrete pulses and the intervals
+between them.
 
-Nope, nothing --- FluxEngine was unable to find any valid data. How about
-1.42us, this time for the whole disk?
-
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0 --manual-clock-rate-us=1.42
-...skipped...
-H.SS Tracks --->
-0. 0 .BBBBBBBBXBBBBBBXXXXXXXXXBB
-0. 1 B.BBBBBBBXXXXBBBXXXXXXXXXXX
-0. 2 B..BBBBBBBBBXBBBXXXXXXXXXXX
-0. 3 B.BBBBBBBXBB.BBBXXXXXXXXXXX
-0. 4 B.BBBBBBBXBB.BBBXBXXXXXXXBB
-0. 5 B.BBBBBBBBBB.BXBXXXXXXXXXBB
-0. 6 B..BBBBBBXBBXBXBXXXXXXXXXXX
-0. 7 B..BBBBBBBBBXB.XXXBBXXXBXBX
-0. 8 B.BBBBBBBBBBXB.BXBBXXXX.XXX
-0. 9 B.BBBBBBBXXXXX.BXXXXXXXXXXX
-0.10 B..BBBBBBBBBXB.BXXXXXXXXXXX
-0.11 B..BBBBBXXBXBB.BXXXXXXXXXXX
-0.12 B.BBBBBB.BXBBB.BXXXXXXXXXXX
-0.13 B..BBBBB.BBBBB.BXXBXXXXXXXX
-0.14 BB.BBBBBXBBBBB.BXBXXXXXXXXX
-0.15 B..BBBBB.XBBBB.BXXXXXXXXXXX
-0.16 B..BBBBBBXBBXBBBXBBXXXXXXXX
-0.17 BBBBBBBB.XBBXBBBXXXXXXXXXXX
-0.18 BBBBXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 42/513 (8%)
-Missing sectors: 234/513 (45%)
-Bad sectors: 237/513 (46%)
-27 tracks, 1 heads, 19 sectors, 512 bytes per sector, 256 kB total
-```
-
-It found something. The sectors in track 0 are now B rather than X, which
-means that FluxEngine at least found them, and look, one sector even passed
-its CRC!
-
-But it turns out that the Victor 9000 actually uses a varying clock rate from
-track to track, so as to fit more data on the longer tracks on the outside of
-the disk. So, manually setting the clock rate to 1.42us has actually made
-things _worse_ at the other end of the disk. Our overall bad sector rate has
-gone up from 20% to 46%.
-
-So, I look back at the histogram. I want to keep using the clock
-autodetection, but persuade it to detect the right clock. There _is_ a peak
-at 1.42us, but there's enough noise around it to confuse the peak detection
-algorithm. You can see from the summary at the end that it thinks the peak
-extends from 1.17us to 3.25us.
-
-The way to correct this is to change the noise floor. This makes it ignore
-frequencies below a certain level. Raising this will make it much more
-conservative about what it considers a frequency peak. With good data, this
-actually makes frequency detection _less_ accurate, but with bad data it can
-help.
-
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0  --show-clock-histogram --noise-floor-factor=0.25
-Reading from: diskimage/:d=0:s=0:t=0
-  0.0: 829 ms in 316193 bytes
-Clock detection histogram:
-1.33  16283 ████████████████████▎
-1.42  22879 ████████████████████████████▌
-1.50   4564 █████▋
-...
-1.67  19594 ████████████████████████▍
-1.75  32059 ████████████████████████████████████████ 
-1.83  18042 ██████████████████████▌
-...
-2.00   8825 ███████████ 
-2.08  16031 ████████████████████ 
-2.17   5080 ██████▎
-...
-...skipped...
-Noise floor:  8014
-Signal level: 3205
-Peak start:   15 (1.25 us)
-Peak end:     18 (1.50 us)
-Median:       17 (1.42 us)
-...skipped...
-Good sectors: 1/19 (5%)
-Missing sectors: 0/19 (0%)
-Bad sectors: 18/19 (94%)
-1 tracks, 1 heads, 19 sectors, 512 bytes per sector, 9 kB total
-```
-
-So, it's now found a peak from 1.25us to 1.50us, with a median of 1.42us ---
-exactly what I wanted. Let's try it on the whole disk:
-
-```
-H.SS Tracks --->
-0. 0 .BBBBBBBBBBBBBBBB.BBB.B..BB.....................................................
-0. 1 B.BBBBBBBBXXX.BBB.BBB.X..BB..........B..........................................
-0. 2 B..BBBBBB.BBXBBBB.BBB.X..BB.......B.............................................
-0. 3 B.BBBBBBBBBB.B.BX.BBB.B..BB......B..............................................
-0. 4 B.BBBBBB.BBB.BBBB.BBB.B..BB.....B...............................................
-0. 5 B.BBBBBBBBBB.BBBB.BBB.XB.BB.....B.B.............................................
-0. 6 B..BBBBBBBBBXB.BX.BBB.X.XXB.....................................................
-0. 7 B..BBBBBBBBBXB.XB.BBB.X.BBB.....................................................
-0. 8 B.BBBBBBBBBBXB.BB.BBB.X.XXX.B...................................................
-0. 9 B.BBBBBBBBXXX..BX.BXB.X.XXXBB...................................................
-0.10 B..BBBBB.BBBXB.BB.BBB...BBB.B...................................................
-0.11 B..BBBBB.BBXBB.BB.BXB.B.BBB.B......B............................................
-0.12 B.BBBBBB.BXBBB.BB.BXX.X.XXX........B...................................XXXXXXXXX
-0.13 B..BBBBB.BBBBB.BX.BBB.X.BXB......BB.........................XXXXXXXXXXXXXXXXXXXX
-0.14 BB.BBBBB..BBBB.BB.BBB.B.XBB.B....BB.B...........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.15 B..BBBBB.BBBBB.BB.BBB.X.XBB.B....B....XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.16 B..BBBBB.BBBXBBBB.BBB.X.BXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.17 BBBBBBBB.BBBX.BBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.18 BBBBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 834/1520 (54%)
-Missing sectors: 346/1520 (22%)
-Bad sectors: 340/1520 (22%)
-80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
-```
-
-54% good sectors --- much better! Most of the top half of the disk is reading
-flawlessly. The bottom half is still dreadful, but much better.
-
-I picked this disk as a sample because it's essentially wrecked. It's the
-worst disk image I've ever seen. Luckily I didn't scan this myself, because
-chances are it's all mouldy and would wreck my disk heads. But its very
-badness makes it a good example.
-
-(I am continually improving the clock detection and data extraction
-algorithms. I have actually seen someone get more data than I off this image,
-with a mostly-good read of track 0. I must find out their secrets...)
+The tool's subject to change without notice as I tinker with it.

doc/technical.md

@@ -13,9 +13,9 @@ Verilog turns the timer, pulse and index information into a bytecode stream
 which encodes intervals, pulses, and whether the index hole has been seen.
 
 This is then streamed back to the PC, where offline software decodes it: it
-does simple statistical analysis to guess the clock rate, then turns the
-pulses into a nice, lined up bit array and from there decodes the file system
-format.
+searches for known bit patterns (which depend on the format), uses those to
+determine the clock rate, and then turns the pulses into a nice, lined up bit
+array and from there decodes the file system format.
 
 Writing back to disk works the same way: bytes are streamed to the
 FluxEngine, where a different datapath state machine thingy (the PSoC5LP has
@@ -24,7 +24,9 @@ stream of pulses to the disk.
 
 The bytecode format represents an interval between pulses as a byte, a pulse
 as a byte, and the index hole as a byte. Timer overflows are handled by
-sending multiple intervals in a row.
+sending multiple intervals in a row. However, the USB transport applies a
+simple compression system to this in order to get the USB bandwidth down to
+something manageable.
 
 An HD floppy has a nominal pulse frequency of 500kHz, and we use a sample
 clock of 12MHz (every 83ns). This means that our 500kHz pulses will have an
@@ -75,7 +77,7 @@ second (one for each 64-byte frame) in order to transfer the data.
 
 The Atmels and STM32s I found were perfectly capable of doing the real-time
 sampling, using hand-tool assembly, but I very much doubt whether they could
-do the USB streaming as well (although I'd like to move away from the Cypress
+do the USB streaming as well (although I want to move away from the Cypress
 onto something less proprietary and easier to source, so I'd like to be
 proven wrong here).
 
@@ -114,9 +116,13 @@ admittedly expensive.)
 
   - [The TEAC FD-05HF-8830 data
     sheet](https://hxc2001.com/download/datasheet/floppy/thirdparty/Teac/TEAC%20FD-05HF-8830.pdf):
-    the technical data sheet for a representative drive. Lots of useful
+    the technical data sheet for a representative 3.5" drive. Lots of useful
     timing numbers here.
 
+  - [The Mitsubishi M4851 data
+    sheet](http://www.bitsavers.org/pdf/mitsubishi/floppy/M4851/TJ2-G30211A_M4851_DSHH_48TPI_OEM_Manual_Nov83.pdf):
+    the equivalent data sheet for a representative 5.25" drive.
+
   - [KryoFlux stream file
     documentation](https://www.kryoflux.com/download/kryoflux_stream_protocol_rev1.1.pdf):
     the format of KryoFlux stream files (partially supported by FluxEngine)

doc/using.md

@@ -46,28 +46,30 @@ In order to do anything useful, you have to plug it in to a floppy disk drive (o
   5. Connect the FluxEngine to your PC via USB --- using the little socket on
      the board, not the big programmer plug.
 
-  6. Insert a scratch disk and do `.obj/fe-rpm` from the shell. The motor
+  6. Insert a scratch disk and do `fluxengine rpm` from the shell. The motor
      should work and it'll tell you that the disk is spinning at about 300
      rpm for a 3.5" disk, or 360 rpm for a 5.25" disk. If it doesn't, please
      [get in touch](https://github.com/davidgiven/fluxengine/issues/new).
 
-  7. Do `.obj/fe-testbulktransport` from the shell. It'll measure your USB
+  7. Do `fluxengine testbulktransport` from the shell. It'll measure your USB
      bandwidth. Ideally you should be getting above 900kB/s. FluxEngine needs
      about 850kB/s, so if you're getting less than this, try a different USB
      port.
 
   8. Insert a standard PC formatted floppy disk into the drive (probably a good
-     idea to remove the old disk first). Then do `.obj/fe-readibm`. It should
-     read the disk, emitting copious diagnostics, and spit out an `ibm.img`
-     file containing the decoded disk image (either 1440kB or 720kB depending).
+     idea to remove the old disk first). Then do `fluxengine read ibm`. It
+     should read the disk, emitting copious diagnostics, and spit out an
+     `ibm.img` file containing the decoded disk image (either 1440kB or 720kB
+     depending).
 
   9. Profit!
 
 ## The programs
 
-I'm sorry to say that the programs are very badly documented --- they're
-moving too quickly for the documentation to keep up. They do all respond to
-`--help`. There are some common properties, described below.
+I'm sorry to say that the client program is very badly documented --- it's
+moving too quickly for the documentation to keep up. It does respond to
+`--help` or `help` depending on context. There are some common properties,
+described below.
 
 ### Source and destination specifiers
 
@@ -76,7 +78,7 @@ use the `--source` (`-s`) and `--dest` (`-d`) options to tell FluxEngine
 which bits of the disk you want to access. These use a common syntax:
 
 ```
-.obj/fe-readibm -s fakedisk.flux:t=0-79:s=0
+fluxengine read ibm -s fakedisk.flux:t=0-79:s=0
 ```
 
   - To access a real disk, leave out the filename (so `:t=0-79:s=0`).
@@ -110,6 +112,25 @@ sensible for the command you're using.
 **Important note:** FluxEngine _always_ uses zero-based units (even if the
 *disk format says otherwise).
 
+### High density disks
+
+High density disks use a different magnetic medium to low and double density
+disks, and have different magnetic properties. 3.5" drives can usually
+autodetect what kind of medium is inserted into the drive based on the hole
+in the disk casing, but 5.25" drives can't. As a result, you need to
+explicitly tell FluxEngine on the command line whether you're using a high
+density disk or not with the `--hd` flag.
+**If you don't do this, your disks may not read correctly and will _certainly_
+fail to write correctly.**
+
+You can distinguish high density 5.25" floppies from the presence of a
+traction ring around the hole in the middle of the disk; if the ring is not
+present, the disk is probably high density. However, this isn't always the
+case, and reading the disk label is much more reliable.
+
+[Lots more information on high density vs double density disks can be found
+here.](http://www.retrotechnology.com/herbs_stuff/guzis.html)
+
 ### The commands
 
 The FluxEngine client software is a largely undocumented set of small tools.
@@ -117,43 +138,47 @@ You'll have to play with them. They all support `--help`. They're not
 installed anywhere and after building you'll find them in the `.obj`
 directory.
 
-  - `fe-erase`: wipes (all or part of) a disk --- erases it without writing
-    a pulsetrain.
+  - `fluxengine erase`: wipes (all or part of) a disk --- erases it without
+  writing a pulsetrain.
 
-  - `fe-inspect`: dumps the raw pulsetrain / bitstream to stdout. Mainly useful
-    for debugging.
+  - `fluxengine inspect`: dumps the raw pulsetrain / bitstream to stdout.
+  Mainly useful for debugging.
 
-  - `fe-read*`: reads various formats of disk. See the per-format documentation
-    linked from the table above. These all take an optional `--write-flux`
-    option which will cause the raw flux to be written to the specified file.
+  - `fluxengine read*`: reads various formats of disk. See the per-format
+  documentation linked from the table above. These all take an optional
+  `--write-flux` option which will cause the raw flux to be written to the
+  specified file.
 
-  - `fe-write*`: writes various formats of disk. Again, see the per-format
-    documentation above.
+  - `fluxengine write*`: writes various formats of disk. Again, see the
+  per-format documentation above.
 
-  - `fe-writeflux`: writes raw flux files. This is much less useful than you
-    might think: you can't write flux files read from a disk to another disk.
-    (See the [FAQ](faq.md) for more information.) It's mainly useful for flux
-    files synthesised by the other `fe-write*` commands.
+  - `fluxengine writeflux`: writes raw flux files. This is much less useful
+  than you might think: you can't write flux files read from a disk to
+  another disk. (See the [FAQ](faq.md) for more information.) It's mainly
+  useful for flux files synthesised by the other `fluxengine write` commands.
 
-  - `fe-writetestpattern`: writes regular pulses (at a configurable interval)
-    to the disk. Useful for testing drive jitter, erasing disks in a more
-    secure fashion, or simply debugging. Goes well with `fe-inspect`.
+  - `fluxengine writetestpattern`: writes regular pulses (at a configurable
+  interval) to the disk. Useful for testing drive jitter, erasing disks in a
+  more secure fashion, or simply debugging. Goes well with `fluxengine
+  inspect`.
 
-  - `fe-rpm`: measures the RPM of the drive (requires a disk in the drive).
-    Mainly useful for testing.
+  - `fluxengine rpm`: measures the RPM of the drive (requires a disk in the
+  drive). Mainly useful for testing.
 
-  - `fe-seek`: moves the head. Mainly useful for finding out whether your drive
-    can seek to track 82. (Mine can't.)
+  - `fluxengine seek`: moves the head. Mainly useful for finding out whether
+  your drive can seek to track 82. (Mine can't.)
 
-  - `fe-testbulktransport`: measures your USB throughput. You need about 600kB/s
-    for FluxEngine to work. You don't need a disk in the drive for this one.
+  - `fluxengine testbulktransport`: measures your USB throughput. You need
+  about 600kB/s for FluxEngine to work. You don't need a disk in the drive
+  for this one.
 
-  - `fe-upgradefluxfile`: occasionally I need to upgrade the flux file format in
-    a non-backwards-compatible way; this tool will upgrade flux files to the new
-    format.
+  - `fluxengine upgradefluxfile`: occasionally I need to upgrade the flux
+  file format in a non-backwards-compatible way; this tool will upgrade flux
+  files to the new format.
 
-Commands which normally take `--source` or `--dest` get a sensible default if left
-unspecified. `fe-readibm` on its own will read drive 0 and write an `ibm.img` file.
+Commands which normally take `--source` or `--dest` get a sensible default if
+left unspecified. `fluxengine read ibm` on its own will read drive 0 and
+write an `ibm.img` file.
 
 ## Extra programs
 
@@ -172,19 +197,27 @@ So you've just received, say, a huge pile of old Brother word processor disks co
 Typically I do this:
 
 ```
-$ fe-readbrother -s :d=0 -o brother.img --write-flux=brother.flux
+$ fluxengine read brother -s :d=0 -o brother.img --write-flux=brother.flux
 ```
 
-This will read the disk in drive 0 and write out a filesystem image. It'll also copy the flux to brother.flux. If I then need to tweak the settings, I can rerun the decode without having to physically touch the disk like this:
+This will read the disk in drive 0 and write out a filesystem image. It'll
+also copy the flux to brother.flux. If I then need to tweak the settings, I
+can rerun the decode without having to physically touch the disk like this:
 
 ```
-$ fe-readbrother -s brother.flux -o brother.img
+$ fluxengine read brother -s brother.flux -o brother.img
 ```
 
-If the disk is particularly fragile, you can force FluxEngine not to retry
+Apart from being drastically faster, this avoids touching the (potentially
+physically fragile) disk.
+
+If the disk is particularly dodgy, you can force FluxEngine not to retry
 failed reads with `--retries=0`. This reduces head movement. **This is not
 recommended.** Floppy disks are inherently unreliable, and the occasional bit
-error is perfectly normal; the sector will read fine next time. If you
-prevent retries, then not only do you get bad sectors in the resulting image,
-but the flux file itself contains the bad read, so attempting a decode of it
-will just reproduce the same bad data.
+error is perfectly normal; FluxEngine will retry and the sector will read
+fine next time. If you prevent retries, then not only do you get bad sectors
+in the resulting image, but the flux file itself contains the bad read, so
+attempting a decode of it will just reproduce the same bad data.
+
+See also the [troubleshooting page](problems.md) for more information about
+reading dubious disks.

lib/aeslanier/aeslanier.h

@@ -3,19 +3,18 @@
 
 #define AESLANIER_RECORD_SEPARATOR 0x55555122
 #define AESLANIER_SECTOR_LENGTH    256
+#define AESLANIER_RECORD_SIZE      (AESLANIER_SECTOR_LENGTH + 5)
 
 class Sector;
 class Fluxmap;
 
-class AesLanierDecoder : public AbstractSoftSectorDecoder
+class AesLanierDecoder : public AbstractDecoder
 {
 public:
     virtual ~AesLanierDecoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-	nanoseconds_t guessClock(Fluxmap& fluxmap) const;
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
 };
 
 #endif

lib/aeslanier/decoder.cc

@@ -1,14 +1,17 @@
 #include "globals.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "aeslanier.h"
 #include "crc.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "sector.h"
 #include "bytes.h"
 #include "record.h"
 #include "fmt/format.h"
 #include <string.h>
 
+static const FluxPattern SECTOR_PATTERN(32, AESLANIER_RECORD_SEPARATOR);
+
 /* This is actually M2FM, rather than MFM, but it our MFM/FM decoder copes fine with it. */
 
 static Bytes reverse_bits(const Bytes& input)
@@ -21,63 +24,42 @@ static Bytes reverse_bits(const Bytes& input)
     return output;
 }
 
-SectorVector AesLanierDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType AesLanierDecoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
+    _sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
+    if (_fmr->eof() || !_sector->clock)
+        return UNKNOWN_RECORD;
+    return SECTOR_RECORD;
+}
+
+void AesLanierDecoder::decodeSectorRecord()
+{
+    /* Skip ID mark. */
+
+    readRawBits(16);
+
+    const auto& rawbits = readRawBits(AESLANIER_RECORD_SIZE*16);
+    const auto& bytes = decodeFmMfm(rawbits).slice(0, AESLANIER_RECORD_SIZE);
+    const auto& reversed = reverse_bits(bytes);
+
+    _sector->logicalTrack = reversed[1];
+    _sector->logicalSide = 0;
+    _sector->logicalSector = reversed[2];
+
+    /* Check header 'checksum' (which seems far too simple to mean much). */
 
-    for (auto& rawrecord : rawRecords)
     {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decodeFmMfm(rawdata);
-        const auto& reversed = reverse_bits(bytes);
+        uint8_t wanted = reversed[3];
+        uint8_t got = reversed[1] + reversed[2];
+        if (wanted != got)
+            return;
+    }
 
-        if (reversed.size() < 0x103)
-            continue;
+    /* Check data checksum, which also includes the header and is
+        * significantly better. */
 
-        unsigned track = reversed[1];
-        unsigned sectorid = reversed[2];
-
-        /* Basic sanity checking. */
-
-        if (track > 85)
-            continue;
-        if (sectorid > 35)
-            continue;
-
-        /* Check header 'checksum' (which seems far too simple to mean much). */
-
-        {
-            uint8_t wanted = reversed[3];
-            uint8_t got = reversed[1] + reversed[2];
-            if (wanted != got)
-                continue;
-        }
-
-        /* Check data checksum, which also includes the header and is
-         * significantly better. */
-
-        Bytes payload = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
-        uint16_t wanted = reversed.reader().seek(0x101).read_le16();
-        uint16_t got = crc16ref(MODBUS_POLY_REF, payload);
-        int status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-        auto sector = std::unique_ptr<Sector>(
-            new Sector(status, track, 0, sectorid, payload));
-        sectors.push_back(std::move(sector));
-	}
-
-	return sectors;
-}
-
-nanoseconds_t AesLanierDecoder::guessClock(Fluxmap& fluxmap) const
-{
-    return fluxmap.guessClock() / 2;
-}
-
-int AesLanierDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffffffff;
-    if (masked == AESLANIER_RECORD_SEPARATOR)
-	 	return 16;
-    return 0;
+    _sector->data = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
+    uint16_t wanted = reversed.reader().seek(0x101).read_le16();
+    uint16_t got = crc16ref(MODBUS_POLY_REF, _sector->data);
+    _sector->status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/amiga/amiga.h

@@ -3,18 +3,18 @@
 
 #define AMIGA_SECTOR_RECORD 0xaaaa44894489LL
 
+#define AMIGA_RECORD_SIZE 0x21f
+
 class Sector;
 class Fluxmap;
 
-class AmigaDecoder : public AbstractSoftSectorDecoder
+class AmigaDecoder : public AbstractDecoder
 {
 public:
     virtual ~AmigaDecoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-	nanoseconds_t guessClock(Fluxmap& fluxmap) const;
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
 };
 
 #endif

lib/amiga/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "amiga.h"
 #include "bytes.h"
@@ -18,6 +19,8 @@
  * MFM works.
  */
          
+static const FluxPattern SECTOR_PATTERN(48, AMIGA_SECTOR_RECORD);
+
 static Bytes deinterleave(const uint8_t*& input, size_t len)
 {
     assert(!(len & 1));
@@ -59,53 +62,36 @@ static uint32_t checksum(const Bytes& bytes)
     return checksum & 0x55555555;
 }
 
-SectorVector AmigaDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType AmigaDecoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& rawbytes = toBytes(rawdata);
-        const auto& bytes = decodeFmMfm(rawdata);
-
-        if (bytes.size() < 544)
-            continue;
-
-        const uint8_t* ptr = bytes.begin() + 4;
-
-        Bytes header = deinterleave(ptr, 4);
-        Bytes recoveryinfo = deinterleave(ptr, 16);
-
-        uint32_t wantedheaderchecksum = deinterleave(ptr, 4).reader().read_be32();
-        uint32_t gotheaderchecksum = checksum(rawbytes.slice(8, 40));
-        if (gotheaderchecksum != wantedheaderchecksum)
-            continue;
-
-        uint32_t wanteddatachecksum = deinterleave(ptr, 4).reader().read_be32();
-        uint32_t gotdatachecksum = checksum(rawbytes.slice(64, 1024));
-        int status = (gotdatachecksum == wanteddatachecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-        Bytes databytes = deinterleave(ptr, 512);
-        unsigned track = header[1] >> 1;
-        unsigned side = header[1] & 1;
-        auto sector = std::unique_ptr<Sector>(
-            new Sector(status, track, side, header[2], databytes));
-        sectors.push_back(std::move(sector));
-	}
-
-	return sectors;
+    _sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
+    if (_fmr->eof() || !_sector->clock)
+        return UNKNOWN_RECORD;
+    return SECTOR_RECORD;
 }
 
-nanoseconds_t AmigaDecoder::guessClock(Fluxmap& fluxmap) const
+void AmigaDecoder::decodeSectorRecord()
 {
-    return fluxmap.guessClock() / 2;
-}
+    const auto& rawbits = readRawBits(AMIGA_RECORD_SIZE*16);
+    const auto& rawbytes = toBytes(rawbits).slice(0, AMIGA_RECORD_SIZE*2);
+    const auto& bytes = decodeFmMfm(rawbits).slice(0, AMIGA_RECORD_SIZE);
 
-int AmigaDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint64_t masked = fifo & 0xffffffffffffULL;
-    if (masked == AMIGA_SECTOR_RECORD)
-		return 64;
-    return 0;
+    const uint8_t* ptr = bytes.begin() + 3;
+
+    Bytes header = deinterleave(ptr, 4);
+    Bytes recoveryinfo = deinterleave(ptr, 16);
+
+    _sector->logicalTrack = header[1] >> 1;
+    _sector->logicalSide = header[1] & 1;
+    _sector->logicalSector = header[2];
+
+    uint32_t wantedheaderchecksum = deinterleave(ptr, 4).reader().read_be32();
+    uint32_t gotheaderchecksum = checksum(rawbytes.slice(6, 40));
+    if (gotheaderchecksum != wantedheaderchecksum)
+        return;
+
+    uint32_t wanteddatachecksum = deinterleave(ptr, 4).reader().read_be32();
+    uint32_t gotdatachecksum = checksum(rawbytes.slice(62, 1024));
+    _sector->data = deinterleave(ptr, 512);
+    _sector->status = (gotdatachecksum == wanteddatachecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/apple2/apple2.h

@@ -10,14 +10,16 @@
 class Sector;
 class Fluxmap;
 
-class Apple2Decoder : public AbstractSoftSectorDecoder
+class Apple2Decoder : public AbstractDecoder
 {
 public:
     virtual ~Apple2Decoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
+
 #endif
 

lib/apple2/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "apple2.h"
 #include "bytes.h"
@@ -10,6 +11,10 @@
 #include <string.h>
 #include <algorithm>
 
+const FluxPattern SECTOR_RECORD_PATTERN(24, APPLE2_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, APPLE2_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
@@ -25,7 +30,7 @@ static int decode_data_gcr(uint8_t gcr)
 /* This is extremely inspired by the MESS implementation, written by Nathan Woods
  * and R. Belmont: https://github.com/mamedev/mame/blob/7914a6083a3b3a8c243ae6c3b8cb50b023f21e0e/src/lib/formats/ap2_dsk.cpp
  */
-static Bytes decode_crazy_data(const uint8_t* inp, int& status)
+static Bytes decode_crazy_data(const uint8_t* inp, Sector::Status& status)
 {
     Bytes output(APPLE2_SECTOR_LENGTH);
 
@@ -60,61 +65,48 @@ uint8_t combine(uint16_t word)
     return word & (word >> 7);
 }
 
-SectorVector Apple2Decoder::decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType Apple2Decoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-    int nextTrack;
-    int nextSector;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const std::vector<bool>& rawdata = rawrecord->data;
-        const Bytes& rawbytes = toBytes(rawdata);
-        ByteReader br(rawbytes);
-
-        if (rawbytes.size() < 8)
-            continue;
-
-        uint32_t signature = br.read_be24();
-        switch (signature)
-        {
-            case APPLE2_SECTOR_RECORD:
-            {
-                uint8_t volume = combine(br.read_be16());
-                nextTrack = combine(br.read_be16());
-                nextSector = combine(br.read_be16());
-                uint8_t checksum = combine(br.read_be16());
-                headerIsValid = checksum == (volume ^ nextTrack ^ nextSector);
-                break;
-            }
-
-            case APPLE2_DATA_RECORD:
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-                    
-                Bytes clipped_bytes = rawbytes.slice(0, APPLE2_ENCODED_SECTOR_LENGTH + 5);
-                int status = Sector::BAD_CHECKSUM;
-                auto data = decode_crazy_data(&clipped_bytes[3], status);
-
-                auto sector = std::unique_ptr<Sector>(
-                    new Sector(status, nextTrack, 0, nextSector, data));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
 }
 
-int Apple2Decoder::recordMatcher(uint64_t fifo) const
+void Apple2Decoder::decodeSectorRecord()
 {
-    uint32_t masked = fifo & 0xffffff;
-    if ((masked == APPLE2_SECTOR_RECORD) || (masked == APPLE2_DATA_RECORD))
-		return 24;
-    return 0;
+    /* Skip ID (as we know it's a APPLE2_SECTOR_RECORD). */
+    readRawBits(24);
+
+    /* Read header. */
+
+    auto header = toBytes(readRawBits(8*8)).slice(0, 8);
+    ByteReader br(header);
+
+    uint8_t volume = combine(br.read_be16());
+    _sector->logicalTrack = combine(br.read_be16());
+    _sector->logicalSector = combine(br.read_be16());
+    uint8_t checksum = combine(br.read_be16());
+    if (checksum == (volume ^ _sector->logicalTrack ^ _sector->logicalSector))
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Apple2Decoder::decodeDataRecord()
+{
+    /* Check ID. */
+
+    Bytes bytes = toBytes(readRawBits(3*8)).slice(0, 3);
+    if (bytes.reader().read_be24() != APPLE2_DATA_RECORD)
+        return;
+
+    /* Read and decode data. */
+
+    unsigned recordLength = APPLE2_ENCODED_SECTOR_LENGTH + 2;
+    bytes = toBytes(readRawBits(recordLength*8)).slice(0, recordLength);
+
+    _sector->status = Sector::BAD_CHECKSUM;
+    _sector->data = decode_crazy_data(&bytes[0], _sector->status);
 }

lib/brother/brother.h

@@ -3,22 +3,23 @@
 
 /* Brother word processor format (or at least, one of them) */
 
-#define BROTHER_SECTOR_RECORD 0xFFFFFD57
-#define BROTHER_DATA_RECORD   0xFFFFFDDB
-#define BROTHER_DATA_RECORD_PAYLOAD  256
-#define BROTHER_DATA_RECORD_CHECKSUM 3
+#define BROTHER_SECTOR_RECORD            0xFFFFFD57
+#define BROTHER_DATA_RECORD              0xFFFFFDDB
+#define BROTHER_DATA_RECORD_PAYLOAD      256
+#define BROTHER_DATA_RECORD_CHECKSUM     3
+#define BROTHER_DATA_RECORD_ENCODED_SIZE 415
 
 class Sector;
 class Fluxmap;
 
-class BrotherDecoder : public AbstractSoftSectorDecoder
+class BrotherDecoder : public AbstractDecoder
 {
 public:
     virtual ~BrotherDecoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
 extern void writeBrotherSectorHeader(std::vector<bool>& bits, unsigned& cursor,

lib/brother/decoder.cc

@@ -1,7 +1,8 @@
 #include "globals.h"
 #include "sql.h"
 #include "fluxmap.h"
-#include "decoders.h"
+#include "decoders/fluxmapreader.h"
+#include "decoders/decoders.h"
 #include "record.h"
 #include "brother.h"
 #include "sector.h"
@@ -9,6 +10,10 @@
 #include "crc.h"
 #include <ctype.h>
 
+const FluxPattern SECTOR_RECORD_PATTERN(32, BROTHER_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(32, BROTHER_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static std::vector<uint8_t> outputbuffer;
 
 /*
@@ -48,89 +53,56 @@ static int decode_header_gcr(uint16_t word)
 	return -1;             
 };
 
-SectorVector BrotherDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType BrotherDecoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-	bool headerIsValid = false;
-	unsigned nextTrack = 0;
-	unsigned nextSector = 0;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
 
-    for (auto& rawrecord : rawRecords)
-    {
-		if (rawrecord->data.size() < 64)
-		{
-			headerIsValid = false;
-			continue;
-		}
+void BrotherDecoder::decodeSectorRecord()
+{
+	readRawBits(32);
+	const auto& rawbits = readRawBits(32);
+	const auto& bytes = toBytes(rawbits).slice(0, 4);
 
-		auto ii = rawrecord->data.cbegin();
-		uint32_t signature = toBytes(ii, ii+32).reader().read_be32();
-		switch (signature)
-		{
-			case BROTHER_SECTOR_RECORD:
-			{
-				headerIsValid = false;
-				if (rawrecord->data.size() < (32+32))
-					break;
+	ByteReader br(bytes);
+	_sector->logicalTrack = decode_header_gcr(br.read_be16());
+	_sector->logicalSector = decode_header_gcr(br.read_be16());
 
-				const auto& by = toBytes(ii+32, ii+64);
-				ByteReader br(by);
-				nextTrack = decode_header_gcr(br.read_be16());
-				nextSector = decode_header_gcr(br.read_be16());
+	/* Sanity check the values read; there's no header checksum and
+		* occasionally we get garbage due to bit errors. */
+	if (_sector->logicalSector > 11)
+		return;
+	if (_sector->logicalTrack > 79)
+		return;
 
-				/* Sanity check the values read; there's no header checksum and
-				 * occasionally we get garbage due to bit errors. */
-				if (nextSector > 11)
-					break;
-				if (nextTrack > 79)
-					break;
+	_sector->status = Sector::DATA_MISSING;
+}
 
-				headerIsValid = true;
-				break;
-			}
+void BrotherDecoder::decodeDataRecord()
+{
+	readRawBits(32);
 
-			case BROTHER_DATA_RECORD:
-			{
-				if (!headerIsValid)
-					break;
+	const auto& rawbits = readRawBits(BROTHER_DATA_RECORD_ENCODED_SIZE*8);
+	const auto& rawbytes = toBytes(rawbits).slice(0, BROTHER_DATA_RECORD_ENCODED_SIZE);
 
-				Bytes rawbytes = toBytes(rawrecord->data.cbegin()+32, rawrecord->data.cend());
-				const int totalsize = BROTHER_DATA_RECORD_PAYLOAD + BROTHER_DATA_RECORD_CHECKSUM;
-
-				Bytes output;
-				ByteWriter bw(output);
-				BitWriter bitw(bw);
-				for (uint8_t b : rawbytes)
-				{
-					uint32_t nibble = decode_data_gcr(b);
-					bitw.push(nibble, 5);
-					if (output.size() == totalsize)
-						break;
-				}
-				bitw.flush();
-				output.resize(totalsize);
-
-				Bytes payload = output.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
-				uint32_t realCrc = crcbrother(payload);
-				uint32_t wantCrc = output.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
-				int status = (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                headerIsValid = false;
-				break;
-			}
-		}
+	Bytes bytes;
+	ByteWriter bw(bytes);
+	BitWriter bitw(bw);
+	for (uint8_t b : rawbytes)
+	{
+		uint32_t nibble = decode_data_gcr(b);
+		bitw.push(nibble, 5);
 	}
+	bitw.flush();
 
-	return sectors;
-}
-
-int BrotherDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffffffff;
-	if ((masked == BROTHER_SECTOR_RECORD) || (masked == BROTHER_DATA_RECORD))
-		return 32;
-    return 0;
+	_sector->data = bytes.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
+	uint32_t realCrc = crcbrother(_sector->data);
+	uint32_t wantCrc = bytes.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
+	_sector->status = (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/brother/encoder.cc

@@ -1,6 +1,6 @@
 #include "globals.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "brother.h"
 #include "crc.h"
 

lib/bytes.cc

@@ -1,6 +1,7 @@
 #include "globals.h"
 #include "bytes.h"
 #include "fmt/format.h"
+#include "common/crunch.h"
 #include <zlib.h>
 
 static std::shared_ptr<std::vector<uint8_t>> createVector(unsigned size)
@@ -124,12 +125,16 @@ uint8_t& Bytes::operator [] (unsigned pos)
 Bytes Bytes::slice(unsigned start, unsigned len) const
 {
     start += _low;
-    boundsCheck(start);
     unsigned end = start + len;
-    if (end > _high)
+    if (start >= _high)
+    {
+        /* Asking for a completely out-of-range slice --- just return zeroes. */
+        return Bytes(len);
+    }
+    else if (end > _high)
     {
         /* Can't share the buffer, as we need to zero-pad the end. */
-        Bytes b(end - start);
+        Bytes b(len);
         std::uninitialized_copy(cbegin()+start, cend(), b.begin());
         return b;
     }
@@ -163,6 +168,23 @@ Bytes toBytes(
     return bytes;
 }
 
+Bytes Bytes::swab() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    ByteReader br(*this);
+
+    while (!br.eof())
+    {
+        uint8_t a = br.read_8();
+        uint8_t b = br.read_8();
+        bw.write_8(b);
+        bw.write_8(a);
+    }
+
+    return output;
+}
+
 Bytes Bytes::compress() const
 {
     uLongf destsize = compressBound(size());
@@ -204,6 +226,60 @@ Bytes Bytes::decompress() const
     return output;
 }
 
+Bytes Bytes::crunch() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    Bytes outputBuffer(1024*1024);
+
+    crunch_state_t cs = {};
+    cs.inputptr = begin();
+    cs.inputlen = size();
+
+    do
+    {
+        cs.outputptr = outputBuffer.begin();
+        cs.outputlen = outputBuffer.size();
+
+        ::crunch(&cs);
+        bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+    }
+    while (cs.inputlen != 0);
+    cs.outputptr = outputBuffer.begin();
+    cs.outputlen = outputBuffer.size();
+    donecrunch(&cs);
+    bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+
+    return output;
+}
+
+Bytes Bytes::uncrunch() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    Bytes outputBuffer(1024*1024);
+
+    crunch_state_t cs = {};
+    cs.inputptr = begin();
+    cs.inputlen = size();
+
+    do
+    {
+        cs.outputptr = outputBuffer.begin();
+        cs.outputlen = outputBuffer.size();
+
+        ::uncrunch(&cs);
+        bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+    }
+    while (cs.inputlen != 0);
+    cs.outputptr = outputBuffer.begin();
+    cs.outputlen = outputBuffer.size();
+    doneuncrunch(&cs);
+    bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+
+    return output;
+}
+
 ByteReader Bytes::reader() const
 {
     return ByteReader(*this);
@@ -214,6 +290,16 @@ ByteWriter Bytes::writer()
     return ByteWriter(*this);
 }
 
+ByteWriter& ByteWriter::operator +=(std::istream& stream)
+{
+    Bytes buffer(4096);
+
+    while (stream.read((char*) buffer.begin(), buffer.size()))
+        this->append(buffer);
+    this->append(buffer.slice(0, stream.gcount()));
+    return *this;
+}
+
 void BitWriter::push(uint32_t bits, size_t size)
 {
     bits <<= 32-size;

lib/bytes.h

@@ -43,9 +43,15 @@ public:
     void adjustBounds(unsigned pos);
     Bytes& resize(unsigned size);
 
+    Bytes& clear()
+    { resize(0); return *this; }
+
     Bytes slice(unsigned start, unsigned len) const;
+    Bytes swab() const;
     Bytes compress() const;
     Bytes decompress() const;
+    Bytes crunch() const;
+    Bytes uncrunch() const;
 
     ByteReader reader() const;
     ByteWriter writer();
@@ -67,7 +73,7 @@ public:
 
     unsigned pos = 0;
     bool eof() const
-    { return pos == _bytes.size(); }
+    { return pos >= _bytes.size(); }
 
     ByteReader& seek(unsigned pos)
     {
@@ -75,6 +81,19 @@ public:
         return *this;
     }
 
+    ByteReader& skip(int delta)
+    {
+        this->pos += delta;
+        return *this;
+    }
+
+    const Bytes read(unsigned len)
+    {
+        const Bytes bytes = _bytes.slice(pos, len);
+        pos += len;
+        return bytes;
+    }
+
     uint8_t read_8()
     {
         return _bytes[pos++];
@@ -247,6 +266,18 @@ public:
         return *this;
     }
 
+    ByteWriter& operator += (std::istream& stream);
+
+    ByteWriter& append(const Bytes data)
+    {
+        return *this += data;
+    }
+
+    ByteWriter& append(std::istream& stream)
+    {
+        return *this += stream;
+    }
+
 private:
     Bytes& _bytes;
 };

lib/c64/c64.h

@@ -1,20 +1,21 @@
 #ifndef C64_H
 #define C64_H
 
-#define C64_RECORD_SEPARATOR 0xfff5
+#define C64_SECTOR_RECORD    0xffd49
+#define C64_DATA_RECORD      0xffd57
 #define C64_SECTOR_LENGTH    256
 
 class Sector;
 class Fluxmap;
 
-class Commodore64Decoder : public AbstractSoftSectorDecoder
+class Commodore64Decoder : public AbstractDecoder
 {
 public:
     virtual ~Commodore64Decoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
 #endif

lib/c64/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "c64.h"
 #include "crc.h"
@@ -11,6 +12,10 @@
 #include <string.h>
 #include <algorithm>
 
+const FluxPattern SECTOR_RECORD_PATTERN(20, C64_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(20, C64_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
@@ -47,67 +52,41 @@ static Bytes decode(const std::vector<bool>& bits)
     return output;
 }
 
-SectorVector Commodore64Decoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType Commodore64Decoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-    unsigned nextSector;
-    unsigned nextTrack;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decode(rawdata);
-
-        if (bytes.size() == 0)
-            continue;
-
-        switch (bytes[0])
-        {
-            case 8: /* sector record */
-            {
-                headerIsValid = false;
-                if (bytes.size() < 6)
-                    break;
-
-                uint8_t checksum = bytes[1];
-                nextSector = bytes[2];
-                nextTrack = bytes[3] - 1;
-                if (checksum != xorBytes(bytes.slice(2, 4)))
-                    break;
-
-                headerIsValid = true;
-                break;
-            }
-            
-            case 7: /* data record */
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-                if (bytes.size() < 258)
-                    break;
-
-                Bytes payload = bytes.slice(1, C64_SECTOR_LENGTH);
-                uint8_t gotChecksum = xorBytes(payload);
-                uint8_t wantChecksum = bytes[257];
-                int status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
 }
 
-int Commodore64Decoder::recordMatcher(uint64_t fifo) const
+void Commodore64Decoder::decodeSectorRecord()
 {
-    uint16_t masked = fifo & 0xffff;
-    if (masked == C64_RECORD_SEPARATOR)
-		return 4;
-    return 0;
+    readRawBits(20);
+
+    const auto& bits = readRawBits(5*10);
+    const auto& bytes = decode(bits).slice(0, 5);
+
+    uint8_t checksum = bytes[0];
+    _sector->logicalSector = bytes[1];
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = bytes[2] - 1;
+    if (checksum == xorBytes(bytes.slice(1, 4)))
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Commodore64Decoder::decodeDataRecord()
+{
+    readRawBits(20);
+
+    const auto& bits = readRawBits(259*10);
+    const auto& bytes = decode(bits).slice(0, 259);
+
+    _sector->data = bytes.slice(0, C64_SECTOR_LENGTH);
+    uint8_t gotChecksum = xorBytes(_sector->data);
+    uint8_t wantChecksum = bytes[256];
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/common/crunch.c

@@ -0,0 +1,86 @@
+#include <stdint.h>
+#include <stdbool.h>
+#include "crunch.h"
+
+void crunch(crunch_state_t* state)
+{
+    while (state->inputlen && state->outputlen)
+    {
+        uint8_t data = *state->inputptr++;
+        state->inputlen--;
+
+        if (data & 0x80)
+        {
+            state->fifo = (state->fifo << 2) | 2 | (data & 1);
+            state->fifolen += 2;
+        }
+        else
+        {
+            state->fifo = (state->fifo << 8) | data;
+            state->fifolen += 8;
+        }
+
+        if (state->fifolen >= 8)
+        {
+            data = state->fifo >> (state->fifolen - 8);
+            *state->outputptr++ = data;
+            state->outputlen--;
+            state->fifolen -= 8;
+        }
+    }
+}
+
+void donecrunch(crunch_state_t* state)
+{
+    if (state->fifolen > 0)
+    {
+        uint8_t b = 0;
+        state->inputptr = &b;
+        state->inputlen = 1;
+        crunch(state);
+    }
+}
+
+void uncrunch(crunch_state_t* state)
+{
+    while (state->inputlen && state->outputlen)
+    {
+        if (state->fifolen < 8)
+        {
+            if (state->inputlen)
+            {
+                state->fifo = (state->fifo << 8) | *state->inputptr++;
+                state->inputlen--;
+                state->fifolen += 8;
+            }
+            else
+                state->fifo <<= 8;
+        }
+
+        uint8_t data = state->fifo >> (state->fifolen - 8);
+        if (data & 0x80)
+        {
+            data = ((data >> 6) & 0x01) | 0x80;
+            state->fifolen -= 2;
+        }
+        else
+            state->fifolen -= 8;
+
+        if (data)
+        {
+            *state->outputptr++ = data;
+            state->outputlen--;
+        }
+    }
+}
+
+void doneuncrunch(crunch_state_t* state)
+{
+    if (state->fifolen > 0)
+    {
+        uint8_t b = 0;
+        state->inputptr = &b;
+        state->inputlen = 1;
+        uncrunch(state);
+    }
+}

lib/common/crunch.h

@@ -0,0 +1,45 @@
+#ifndef CRUNCH_H
+#define CRUNCH_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/* To save bandwidth, we compress the byte stream from the sampler when
+ * sending it over USB. The encoding used is:
+ * 
+ * 0nnn.nnnn: value 0x00..0x7f
+ * 1n       : value 0x80|n
+ *
+ * The end of the buffer is terminated with zeroes, which are ignored
+ * (not written to the output).
+ * 
+ * This saves ~40%, which gets us in under the bandwidth cap.
+ */
+    
+typedef struct crunch_state_t
+{
+    const uint8_t* inputptr;
+    uint32_t inputlen;
+    uint8_t* outputptr;
+    uint32_t outputlen;
+    uint16_t fifo;
+    uint8_t fifolen;
+}
+crunch_state_t;
+
+/* Crunches as much as possible and then stops. */
+extern void crunch(crunch_state_t* state);
+extern void donecrunch(crunch_state_t* state);
+
+/* Uncrunches as much as possible and then stops. */
+extern void uncrunch(crunch_state_t* state);
+extern void doneuncrunch(crunch_state_t* state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

lib/crc.cc

@@ -2,6 +2,47 @@
 #include "bytes.h"
 #include "crc.h"
 
+template <class T>
+T reflect(T bin, unsigned width = sizeof(T)*8)
+{
+	T bout = 0;
+	while (width--)
+	{
+		bout <<= 1;
+		bout |= (bin & 1);
+		bin >>= 1;
+	}
+	return bout;
+}
+
+uint64_t generic_crc(const struct crcspec& spec, const Bytes& bytes)
+{
+	uint64_t crc = spec.init;
+	uint64_t top = 1LL << (spec.width-1);
+	uint64_t mask = (top<<1) - 1;
+
+	for (uint8_t b : bytes)
+	{
+		if (spec.refin)
+			b = reflect(b);
+
+		for (uint8_t i = 0x80; i != 0; i >>= 1)
+		{
+			uint64_t bit = crc & top;
+			crc <<= 1;
+			if (b & i)
+				bit ^= top;
+			if (bit)
+				crc ^= spec.poly;
+		}
+	}
+
+	if (spec.refout)
+		crc = reflect(crc, spec.width);
+	crc ^= spec.xorout;
+	return crc & mask;
+}
+
 uint16_t sumBytes(const Bytes& bytes)
 {
 	ByteReader br(bytes);
@@ -36,11 +77,10 @@ uint16_t crc16(uint16_t poly, uint16_t crc, const Bytes& bytes)
 	return crc;
 }
 
-uint16_t crc16ref(uint16_t poly, const Bytes& bytes)
+uint16_t crc16ref(uint16_t poly, uint16_t crc, const Bytes& bytes)
 {
 	ByteReader br(bytes);
 
-	uint16_t crc = 0xffff;
 	while (!br.eof())
 	{
 		crc ^= br.read_8();

lib/crc.h

@@ -6,14 +6,29 @@
 #define MODBUS_POLY_REF 0xa001
 #define BROTHER_POLY    0x000201
 
+struct crcspec
+{
+    unsigned width;
+    uint64_t poly;
+    uint64_t init;
+    uint64_t xorout;
+    bool refin;
+    bool refout;
+};
+
+extern uint64_t generic_crc(const struct crcspec& spec, const Bytes& bytes);
+
 extern uint16_t sumBytes(const Bytes& bytes);
 extern uint8_t xorBytes(const Bytes& bytes);
 extern uint16_t crc16(uint16_t poly, uint16_t init, const Bytes& bytes);
-extern uint16_t crc16ref(uint16_t poly, const Bytes& bytes);
+extern uint16_t crc16ref(uint16_t poly, uint16_t init, const Bytes& bytes);
 extern uint32_t crcbrother(const Bytes& bytes);
 
 static inline uint16_t crc16(uint16_t poly, const Bytes& bytes)
 { return crc16(poly, 0xffff, bytes); }
 
+static inline uint16_t crc16ref(uint16_t poly, const Bytes& bytes)
+{ return crc16ref(poly, 0xffff, bytes); }
+
 #endif
 

lib/dataspec.cc

@@ -5,9 +5,6 @@
 #include <regex>
 #include <sstream>
 
-static const std::regex MOD_REGEX("([a-z]*)=([-x+0-9,]*)");
-static const std::regex DATA_REGEX("([0-9]+)(?:(?:-([0-9]+))|(?:\\+([0-9]+)))?(?:x([0-9]+))?");
-
 std::vector<std::string> DataSpec::split(
         const std::string& s, const std::string& delimiter)
 {
@@ -30,6 +27,9 @@ std::vector<std::string> DataSpec::split(
 
 DataSpec::Modifier DataSpec::parseMod(const std::string& spec)
 {
+    static const std::regex MOD_REGEX("([a-z]*)=([-x+0-9,]*)");
+    static const std::regex DATA_REGEX("([0-9]+)(?:(?:-([0-9]+))|(?:\\+([0-9]+)))?(?:x([0-9]+))?");
+
     std::smatch match;
     if (!std::regex_match(spec, match, MOD_REGEX))
         Error() << "invalid data modifier syntax '" << spec << "'";

lib/dataspec.h

@@ -49,7 +49,7 @@ public:
     unsigned revolutions;
 };
 
-std::ostream& operator << (std::ostream& os, const DataSpec& dataSpec)
+static inline std::ostream& operator << (std::ostream& os, const DataSpec& dataSpec)
 { os << (std::string)dataSpec; return os; }
 
 class DataSpecFlag : public Flag
@@ -58,15 +58,21 @@ public:
     DataSpecFlag(const std::vector<std::string>& names, const std::string helptext,
             const std::string& defaultValue):
         Flag(names, helptext),
-        value(defaultValue)
+        _value(defaultValue)
     {}
 
-    bool hasArgument() const { return true; }
-    const std::string defaultValueAsString() const { return value; }
-    void set(const std::string& value) { this->value.set(value); }
+    const DataSpec& get() const
+    { checkInitialised(); return _value; }
 
-public:
-    DataSpec value;
+    operator const DataSpec& () const
+    { return get(); }
+
+    bool hasArgument() const { return true; }
+    const std::string defaultValueAsString() const { return _value; }
+    void set(const std::string& value) { _value.set(value); }
+
+private:
+    DataSpec _value;
 };
 
 #endif

lib/decoders/decoders.cc

@@ -1,299 +1,79 @@
 #include "globals.h"
 #include "flags.h"
 #include "fluxmap.h"
-#include "decoders.h"
+#include "decoders/fluxmapreader.h"
+#include "decoders/decoders.h"
 #include "record.h"
 #include "protocol.h"
-#include "rawbits.h"
+#include "decoders/rawbits.h"
+#include "track.h"
+#include "sector.h"
 #include "fmt/format.h"
 #include <numeric>
 
-static DoubleFlag clockDecodeThreshold(
-    { "--clock-decode-threshold" },
-    "Pulses below this fraction of a clock tick are considered spurious and ignored.",
-    0.80);
-
-static SettableFlag showClockHistogram(
-    { "--show-clock-histogram" },
-    "Dump the clock detection histogram.");
-
-static DoubleFlag manualClockRate(
-	{ "--manual-clock-rate-us" },
-	"If not zero, force this clock rate; if zero, try to autodetect it.",
-	0.0);
-
-static DoubleFlag noiseFloorFactor(
-    { "--noise-floor-factor" },
-    "Clock detection noise floor (min + (max-min)*factor).",
-    0.01);
-
-static DoubleFlag signalLevelFactor(
-    { "--signal-level-factor" },
-    "Clock detection signal level (min + (max-min)*factor).",
-    0.05);
-
-static const std::string BLOCK_ELEMENTS[] =
-{ " ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█" };
-
-/* 
-* Tries to guess the clock by finding the smallest common interval.
- * Returns nanoseconds.
- */
-nanoseconds_t Fluxmap::guessClock() const
+void AbstractDecoder::decodeToSectors(Track& track)
 {
-	if (manualClockRate != 0.0)
-		return manualClockRate * 1000.0;
+    Sector sector;
+    sector.physicalSide = track.physicalSide;
+    sector.physicalTrack = track.physicalTrack;
+    FluxmapReader fmr(*track.fluxmap);
 
-    uint32_t buckets[256] = {};
-    size_t cursor = 0;
-    FluxmapReader fr(*this);
-    while (cursor < bytes())
-    {
-        unsigned interval;
-        int opcode = fr.readPulse(interval);
-        if (opcode != 0x80)
-            break;
-        if (interval > 0xff)
-            continue;
-        buckets[interval]++;
-    }
-    
-    uint32_t max = *std::max_element(std::begin(buckets), std::end(buckets));
-    uint32_t min = *std::min_element(std::begin(buckets), std::end(buckets));
-    uint32_t noise_floor = min + (max-min)*noiseFloorFactor;
-    uint32_t signal_level = min + (max-min)*signalLevelFactor;
+    _track = &track;
+    _sector = &sector;
+    _fmr = &fmr;
 
-    /* Find a point solidly within the first pulse. */
-
-    int pulseindex = 0;
-    while (pulseindex < 256)
-    {
-        if (buckets[pulseindex] > signal_level)
-            break;
-        pulseindex++;
-    }
-    if (pulseindex == -1)
-        return 0;
-
-    /* Find the upper and lower bounds of the pulse. */
-
-    int peaklo = pulseindex;
-    while (peaklo > 0)
-    {
-        if (buckets[peaklo] < noise_floor)
-            break;
-        peaklo--;
-    }
-
-    int peakhi = pulseindex;
-    while (peakhi < 255)
-    {
-        if (buckets[peakhi] < noise_floor)
-            break;
-        peakhi++;
-    }
-
-    /* Find the total accumulated size of the pulse. */
-
-    uint32_t total_size = 0;
-    for (int i = peaklo; i < peakhi; i++)
-        total_size += buckets[i];
-
-    /* Now find the median. */
-
-    uint32_t count = 0;
-    int median = peaklo;
-    while (median < peakhi)
-    {
-        count += buckets[median];
-        if (count > (total_size/2))
-            break;
-        median++;
-    }
-
-    if (showClockHistogram)
-    {
-        std::cout << "Clock detection histogram:" << std::endl;
-
-		bool skipping = true;
-        for (int i=0; i<256; i++)
-		{
-			uint32_t value = buckets[i];
-			if (value < noise_floor/2)
-			{
-				if (!skipping)
-					std::cout << "..." << std::endl;
-				skipping = true;
-			}
-			else
-			{
-				skipping = false;
-
-				int bar = 320*value/max;
-				int fullblocks = bar / 8;
-
-				std::string s;
-				for (int j=0; j<fullblocks; j++)
-					s += BLOCK_ELEMENTS[8];
-				s += BLOCK_ELEMENTS[bar & 7];
-
-				std::cout << fmt::format("{:.2f} {:6} {}", (double)i * US_PER_TICK, value, s);
-				std::cout << std::endl;
-			}
-		}
-
-        std::cout << fmt::format("Noise floor:  {}", noise_floor) << std::endl;
-        std::cout << fmt::format("Signal level: {}", signal_level) << std::endl;
-        std::cout << fmt::format("Peak start:   {} ({:.2f} us)", peaklo, peaklo*US_PER_TICK) << std::endl;
-        std::cout << fmt::format("Peak end:     {} ({:.2f} us)", peakhi, peakhi*US_PER_TICK) << std::endl;
-        std::cout << fmt::format("Median:       {} ({:.2f} us)", median, median*US_PER_TICK) << std::endl;
-    }
-
-    /* 
-     * Okay, the median should now be a good candidate for the (or a) clock.
-     * How this maps onto the actual clock rate depends on the encoding.
-     */
-
-    return median * NS_PER_TICK;
-}
-
-/* Decodes a fluxmap into a nice aligned array of bits. */
-const RawBits Fluxmap::decodeToBits(nanoseconds_t clockPeriod) const
-{
-    int pulses = duration() / clockPeriod;
-    nanoseconds_t lowerThreshold = clockPeriod * clockDecodeThreshold;
-
-    auto bitmap = std::make_unique<std::vector<bool>>(pulses);
-    auto indices = std::make_unique<std::vector<size_t>>();
-
-    unsigned count = 0;
-    nanoseconds_t timestamp = 0;
-    FluxmapReader fr(*this);
+    beginTrack();
     for (;;)
     {
-        for (;;)
-        {
-            unsigned interval;
-            int opcode = fr.read(interval);
-            timestamp += interval * NS_PER_TICK;
-            if (opcode == -1)
-                goto abort;
-            else if ((opcode == 0x80) && (timestamp >= lowerThreshold))
-                break;
-            else if (opcode == 0x81)
-                indices->push_back(count);
-        }
-
-        int clocks = (timestamp + clockPeriod/2) / clockPeriod;
-        count += clocks;
-        if (count >= bitmap->size())
-            goto abort;
-        bitmap->at(count) = true;
-        timestamp = 0;
-    }
-abort:
-
-    RawBits rawbits(std::move(bitmap), std::move(indices));
-    return rawbits;
-}
-
-nanoseconds_t AbstractDecoder::guessClock(Fluxmap& fluxmap) const
-{
-    return fluxmap.guessClock();
-}
-
-RawRecordVector AbstractSoftSectorDecoder::extractRecords(const RawBits& rawbits) const
-{
-    RawRecordVector records;
-    uint64_t fifo = 0;
-    size_t cursor = 0;
-    int matchStart = -1;
-
-    auto pushRecord = [&](size_t end)
-    {
-        if (matchStart == -1)
+        Fluxmap::Position recordStart = sector.position = fmr.tell();
+        sector.clock = 0;
+        sector.status = Sector::MISSING;
+        sector.data.clear();
+        sector.logicalSector = sector.logicalSide = sector.logicalTrack = 0;
+        RecordType r = advanceToNextRecord();
+        if (fmr.eof() || !sector.clock)
             return;
-
-        records.push_back(
-            std::unique_ptr<RawRecord>(
-                new RawRecord(
-                    matchStart,
-                    rawbits.begin() + matchStart,
-                    rawbits.begin() + end)
-            )
-        );
-    };
-
-    while (cursor < rawbits.size())
-    {
-        fifo = (fifo << 1) | rawbits[cursor++];
-        
-        int match = recordMatcher(fifo);
-        if (match > 0)
+        if ((r == UNKNOWN_RECORD) || (r == DATA_RECORD))
         {
-            pushRecord(cursor - match);
-            matchStart = cursor - match;
+            fmr.readNextMatchingOpcode(F_OP_PULSE);
+            continue;
         }
+
+        /* Read the sector record. */
+
+        recordStart = fmr.tell();
+        decodeSectorRecord();
+        pushRecord(recordStart, fmr.tell());
+        if (sector.status == Sector::DATA_MISSING)
+        {
+            /* The data is in a separate record. */
+
+            r = advanceToNextRecord();
+            if (r == DATA_RECORD)
+            {
+                recordStart = fmr.tell();
+                decodeDataRecord();
+                pushRecord(recordStart, fmr.tell());
+            }
+        }
+
+        if (sector.status != Sector::MISSING)
+            track.sectors.push_back(sector);
     }
-    pushRecord(cursor);
-    
-    return records;
 }
 
-RawRecordVector AbstractHardSectorDecoder::extractRecords(const RawBits& rawbits) const
+void AbstractDecoder::pushRecord(const Fluxmap::Position& start, const Fluxmap::Position& end)
 {
-    /* This is less easy than it looks.
-     *
-     * Hard-sectored disks contain one extra index hole, marking the top of the
-     * disk. This appears halfway in between two start-of-sector index holes. We
-     * need to find this and ignore it, otherwise we'll split that sector in two
-     * (and it won't work).
-     * 
-     * The routine here is pretty simple and requires this extra hole to have
-     * valid index holes either side of it --- it can't cope with the extra
-     * index hole being the first or last seen. Always give it slightly more
-     * than one revolution of data.
-     */
+    Fluxmap::Position here = _fmr->tell();
 
-    RawRecordVector records;
-    const auto& indices = rawbits.indices();
-    if (!indices.empty())
-    {
-        unsigned total = 0;
-        unsigned previous = 0;
-        for (unsigned index : indices)
-        {
-            total += index - previous;
-            previous = index;
-        }
-        total += rawbits.size() - previous;
+    RawRecord record;
+    record.physicalSide = _track->physicalSide;
+    record.physicalTrack = _track->physicalTrack;
+    record.clock = _sector->clock;
+    record.position = start;
 
-        unsigned sectors_must_be_bigger_than = (total / indices.size()) * 2/3;
-
-
-        previous = 0;
-        auto pushRecord = [&](size_t end)
-        {
-            if ((end - previous) < sectors_must_be_bigger_than)
-                return;
-
-            records.push_back(
-                std::unique_ptr<RawRecord>(
-                    new RawRecord(
-                        previous,
-                        rawbits.begin() + previous,
-                        rawbits.begin() + end)
-                )
-            );
-            previous = end;
-        };
-
-        for (unsigned index : indices)
-            pushRecord(index);
-        pushRecord(rawbits.size());
-    }
-
-    return records;
+    _fmr->seek(start);
+    record.data = toBytes(_fmr->readRawBits(end, _sector->clock));
+    _track->rawrecords.push_back(record);
+    _fmr->seek(here);
 }
-
-

lib/decoders/decoders.h

@@ -2,15 +2,22 @@
 #define DECODERS_H
 
 #include "bytes.h"
+#include "sector.h"
+#include "record.h"
+#include "decoders/fluxmapreader.h"
 
 class Sector;
 class Fluxmap;
+class FluxmapReader;
 class RawRecord;
 class RawBits;
+class Track;
 
 typedef std::vector<std::unique_ptr<RawRecord>> RawRecordVector;
 typedef std::vector<std::unique_ptr<Sector>> SectorVector;
 
+extern void setDecoderManualClockRate(double clockrate_us);
+
 extern Bytes decodeFmMfm(std::vector<bool>::const_iterator start,
     std::vector<bool>::const_iterator end);
 
@@ -22,28 +29,36 @@ class AbstractDecoder
 public:
     virtual ~AbstractDecoder() {}
 
-    virtual nanoseconds_t guessClock(Fluxmap& fluxmap) const;
-    virtual RawRecordVector extractRecords(const RawBits& rawbits) const = 0;
-    virtual SectorVector decodeToSectors(const RawRecordVector& rawrecords,
-            unsigned physicalTrack) = 0;
-};
-
-class AbstractSoftSectorDecoder : public AbstractDecoder
-{
 public:
-    virtual ~AbstractSoftSectorDecoder() {}
+    enum RecordType
+    {
+        SECTOR_RECORD,
+        DATA_RECORD,
+        UNKNOWN_RECORD
+    };
 
-    RawRecordVector extractRecords(const RawBits& rawbits) const;
-
-    virtual int recordMatcher(uint64_t fifo) const = 0;
-};
-
-class AbstractHardSectorDecoder : public AbstractDecoder
-{
 public:
-    virtual ~AbstractHardSectorDecoder() {}
+    void decodeToSectors(Track& track);
+    void pushRecord(const Fluxmap::Position& start, const Fluxmap::Position& end);
 
-    RawRecordVector extractRecords(const RawBits& bits) const;
+    std::vector<bool> readRawBits(unsigned count)
+    { return _fmr->readRawBits(count, _sector->clock); }
+
+    Fluxmap::Position tell()
+    { return _fmr->tell(); } 
+
+    void seek(const Fluxmap::Position& pos)
+    { return _fmr->seek(pos); } 
+
+protected:
+    virtual void beginTrack() {};
+    virtual RecordType advanceToNextRecord() = 0;
+    virtual void decodeSectorRecord() = 0;
+    virtual void decodeDataRecord() {};
+
+    FluxmapReader* _fmr;
+    Track* _track;
+    Sector* _sector;
 };
 
 #endif

lib/decoders/fluxmapreader.cc

@@ -0,0 +1,284 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "flags.h"
+#include "protocol.h"
+#include "fmt/format.h"
+#include <numeric>
+#include <math.h>
+#include <strings.h>
+
+FlagGroup fluxmapReaderFlags;
+
+DoubleFlag pulseDebounceThreshold(
+    { "--pulse-debounce-threshold" },
+    "Ignore pulses with intervals short than this, in fractions of a clock.",
+    0.30);
+
+static DoubleFlag clockDecodeThreshold(
+    { "--bit-error-threshold" },
+    "Amount of error to tolerate in pulse timing, in fractions of a clock.",
+    0.20);
+
+static DoubleFlag clockIntervalBias(
+    { "--clock-interval-bias" },
+    "Adjust intervals between pulses by this many clocks before decoding.",
+    -0.02);
+
+int FluxmapReader::readOpcode(unsigned& ticks)
+{
+    ticks = 0;
+
+    while (!eof())
+    {
+        uint8_t b = _bytes[_pos.bytes++];
+        if (b < 0x80)
+            ticks += b;
+        else
+        {
+            _pos.ticks += ticks;
+            return b;
+        }
+    }
+
+    _pos.ticks += ticks;
+    return -1;
+}
+
+unsigned FluxmapReader::readNextMatchingOpcode(uint8_t opcode)
+{
+    unsigned ticks = 0;
+
+    for (;;)
+    {
+        unsigned thisTicks;
+        int op = readOpcode(thisTicks);
+        ticks += thisTicks;
+        if (op == -1)
+            return 0;
+        if (op == opcode)
+            return ticks;
+    }
+}
+
+unsigned FluxmapReader::readInterval(nanoseconds_t clock)
+{
+    unsigned thresholdTicks = (clock * pulseDebounceThreshold) / NS_PER_TICK;
+    unsigned ticks = 0;
+
+    while (ticks < thresholdTicks)
+    {
+        unsigned thisTicks = readNextMatchingOpcode(F_OP_PULSE);
+        if (!thisTicks)
+            break;
+        ticks += thisTicks;
+    }
+    return ticks;
+}
+
+static int findLowestSetBit(uint64_t value)
+{
+    if (!value)
+        return 0;
+    int bit = 1;
+    while (!(value & 1))
+    {
+        value >>= 1;
+        bit++;
+    }
+    return bit;
+}
+
+FluxPattern::FluxPattern(unsigned bits, uint64_t pattern):
+    _bits(bits)
+{
+    const uint64_t TOPBIT = 1ULL << 63;
+
+    assert(pattern != 0);
+
+    unsigned lowbit = findLowestSetBit(pattern)-1;
+
+    pattern <<= 64 - bits;
+    _highzeroes = 0;
+    while (!(pattern & TOPBIT))
+    {
+        pattern <<= 1;
+        _highzeroes++;
+    }
+
+    _length = 0;
+    while (pattern != TOPBIT)
+    {
+        unsigned interval = 0;
+        do
+        {
+            pattern <<= 1;
+            interval++;
+        }
+        while (!(pattern & TOPBIT));
+        _intervals.push_back(interval);
+        _length += interval;
+    }
+
+    if (lowbit)
+    {
+        _lowzero = true;
+        /* Note that length does *not* include this interval. */
+        _intervals.push_back(lowbit + 1);
+    }
+}
+
+bool FluxPattern::matches(const unsigned* end, FluxMatch& match) const
+{
+    const unsigned* start = end - _intervals.size();
+    unsigned candidatelength = std::accumulate(start, end - _lowzero, 0);
+    if (!candidatelength)
+        return false;
+    match.clock = (double)candidatelength / (double)_length;
+
+    unsigned exactIntervals = _intervals.size() - _lowzero;
+    for (unsigned i=0; i<exactIntervals; i++)
+    {
+        double ii = match.clock * (double)_intervals[i];
+        double ci = (double)start[i];
+        double error = fabs((ii - ci) / match.clock);
+        if (error > clockDecodeThreshold)
+            return false;
+    }
+
+    if (_lowzero)
+    {
+        double ii = match.clock * (double)_intervals[exactIntervals];
+        double ci = (double)start[exactIntervals];
+        double error = (ii - ci) / match.clock;
+        if (error > clockDecodeThreshold)
+            return false;
+    }
+
+    match.matcher = this;
+    match.intervals = _intervals.size();
+    match.zeroes = _highzeroes;
+    return true;
+}
+
+FluxMatchers::FluxMatchers(const std::initializer_list<const FluxMatcher*> matchers):
+    _matchers(matchers)
+{
+    _intervals = 0;
+    for (const auto* matcher : matchers)
+        _intervals = std::max(_intervals, matcher->intervals());
+}
+
+bool FluxMatchers::matches(const unsigned* intervals, FluxMatch& match) const
+{
+    for (const auto* matcher : _matchers)
+    {
+        if (matcher->matches(intervals, match))
+            return true;
+    }
+    return false;
+}
+
+void FluxmapReader::seek(nanoseconds_t ns)
+{
+    unsigned ticks = ns / NS_PER_TICK;
+    if (ticks < _pos.ticks)
+    {
+        _pos.ticks = 0;
+        _pos.bytes = 0;
+    }
+
+    while (!eof() && (_pos.ticks < ticks))
+    {
+        unsigned t;
+        readOpcode(t);
+    }
+    _pos.zeroes = 0;
+}
+
+nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern)
+{
+    const FluxMatcher* unused;
+    return seekToPattern(pattern, unused);
+}
+
+nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern, const FluxMatcher*& matching)
+{
+    unsigned intervalCount = pattern.intervals();
+    unsigned candidates[intervalCount+1];
+    Fluxmap::Position positions[intervalCount+1];
+
+    for (unsigned i=0; i<=intervalCount; i++)
+    {
+        positions[i] = tell();
+        candidates[i] = 0;
+    }
+
+    while (!eof())
+    {
+        FluxMatch match;
+        if (pattern.matches(&candidates[intervalCount+1], match))
+        {
+            seek(positions[intervalCount-match.intervals]);
+            _pos.zeroes = match.zeroes;
+            matching = match.matcher;
+            return match.clock * NS_PER_TICK;
+        }
+
+        for (unsigned i=0; i<intervalCount; i++)
+        {
+            positions[i] = positions[i+1];
+            candidates[i] = candidates[i+1];
+        }
+        candidates[intervalCount] = readNextMatchingOpcode(F_OP_PULSE);
+        positions[intervalCount] = tell();
+
+    }
+
+    matching = NULL;
+    return 0;
+}
+
+void FluxmapReader::seekToIndexMark()
+{
+    readNextMatchingOpcode(F_OP_INDEX);
+    _pos.zeroes = 0;
+}
+
+bool FluxmapReader::readRawBit(nanoseconds_t clockPeriod)
+{
+    assert(clockPeriod != 0);
+
+    if (_pos.zeroes)
+    {
+        _pos.zeroes--;
+        return false;
+    }
+
+    nanoseconds_t interval = readInterval(clockPeriod)*NS_PER_TICK;
+    double clocks = (double)interval / clockPeriod + clockIntervalBias;
+
+    if (clocks < 1.0)
+        clocks = 1.0;
+    _pos.zeroes = (int)round(clocks) - 1;
+    return true;
+}
+
+std::vector<bool> FluxmapReader::readRawBits(unsigned count, nanoseconds_t clockPeriod)
+{
+    std::vector<bool> result;
+    while (!eof() && count--)
+    {
+        bool b = readRawBit(clockPeriod);
+        result.push_back(b);
+    }
+    return result;
+}
+
+std::vector<bool> FluxmapReader::readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod)
+{
+    std::vector<bool> result;
+    while (!eof() && (_pos.bytes < until.bytes))
+        result.push_back(readRawBit(clockPeriod));
+    return result;
+}

lib/decoders/fluxmapreader.h

@@ -0,0 +1,121 @@
+#ifndef FLUXMAPREADER_H
+#define FLUXMAPREADER_H
+
+#include "fluxmap.h"
+#include "protocol.h"
+#include "flags.h"
+
+extern FlagGroup fluxmapReaderFlags;
+
+class FluxMatcher;
+
+struct FluxMatch
+{
+    const FluxMatcher* matcher;
+    unsigned intervals;
+    double clock;
+    unsigned zeroes;
+};
+
+class FluxMatcher
+{
+public:
+    virtual ~FluxMatcher() {}
+
+    /* Returns the number of intervals matched */
+    virtual bool matches(const unsigned* intervals, FluxMatch& match) const = 0;
+    virtual unsigned intervals() const = 0;
+};
+
+class FluxPattern : public FluxMatcher
+{
+public:
+    FluxPattern(unsigned bits, uint64_t patterns);
+
+    bool matches(const unsigned* intervals, FluxMatch& match) const override;
+
+    unsigned intervals() const override
+    { return _intervals.size(); }
+
+private:
+    std::vector<unsigned> _intervals;
+    unsigned _length;
+    unsigned _bits;
+    unsigned _highzeroes;
+    bool _lowzero = false;
+
+public:
+    friend void test_patternconstruction();
+    friend void test_patternmatching();
+};
+
+class FluxMatchers : public FluxMatcher
+{
+public:
+    FluxMatchers(const std::initializer_list<const FluxMatcher*> matchers);
+
+    bool matches(const unsigned* intervals, FluxMatch& match) const override;
+
+    unsigned intervals() const override
+    { return _intervals; }
+
+private:
+    unsigned _intervals;
+    std::vector<const FluxMatcher*> _matchers;
+};
+
+class FluxmapReader
+{
+public:
+    FluxmapReader(const Fluxmap& fluxmap):
+        _fluxmap(fluxmap),
+        _bytes(fluxmap.ptr()),
+        _size(fluxmap.bytes())
+    {
+        rewind();
+    }
+
+    FluxmapReader(const Fluxmap&& fluxmap) = delete;
+
+    void rewind()
+    {
+        _pos.bytes = 0;
+        _pos.ticks = 0;
+        _pos.zeroes = 0;
+    }
+
+    bool eof() const
+    { return _pos.bytes == _size; }
+
+    Fluxmap::Position tell() const
+    { return _pos; }
+
+    /* Important! You can only reliably seek to 1 bits. */
+    void seek(const Fluxmap::Position& pos)
+    {
+        _pos = pos;
+    }
+
+    int readOpcode(unsigned& ticks);
+    unsigned readNextMatchingOpcode(uint8_t opcode);
+    unsigned readInterval(nanoseconds_t clock); /* with debounce support */
+
+    /* Important! You can only reliably seek to 1 bits. */
+    void seek(nanoseconds_t ns);
+
+    void seekToIndexMark();
+    nanoseconds_t seekToPattern(const FluxMatcher& pattern);
+    nanoseconds_t seekToPattern(const FluxMatcher& pattern, const FluxMatcher*& matching);
+
+    bool readRawBit(nanoseconds_t clockPeriod);
+    std::vector<bool> readRawBits(unsigned count, nanoseconds_t clockPeriod);
+    std::vector<bool> readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod);
+
+private:
+    const Fluxmap& _fluxmap;
+    const uint8_t* _bytes;
+    const size_t _size;
+    Fluxmap::Position _pos;
+};
+
+#endif

lib/decoders/fluxreader/hardwarefluxreader.cc.orig

@@ -9,6 +9,13 @@ static IntFlag revolutions(
     "read this many revolutions of the disk",
     1);
 
+static bool high_density = false;
+
+void setHardwareFluxReaderDensity(bool high_density)
+{
+	::high_density = high_density;
+}
+
 class HardwareFluxReader : public FluxReader
 {
 public:
@@ -24,9 +31,12 @@ public:
 public:
     std::unique_ptr<Fluxmap> readFlux(int track, int side)
     {
-        usbSetDrive(_drive);
+        usbSetDrive(_drive, high_density);
         usbSeek(track);
-        return usbRead(side, revolutions);
+        Bytes crunched = usbRead(side, revolutions);
+        auto fluxmap = std::make_unique<Fluxmap>();
+        fluxmap->appendBytes(crunched.uncrunch());
+        return fluxmap;
     }
 
     void recalibrate()

lib/decoders/fmmfm.cc

@@ -1,5 +1,5 @@
 #include "globals.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 
 Bytes decodeFmMfm(
         std::vector<bool>::const_iterator ii, std::vector<bool>::const_iterator end)
@@ -26,7 +26,7 @@ Bytes decodeFmMfm(
     ByteWriter bw(bytes);
 
     int bitcount = 0;
-    uint8_t fifo;
+    uint8_t fifo = 0;
 
     while (ii != end)
     {

lib/encoder.cc

@@ -12,6 +12,7 @@ Fluxmap& Fluxmap::appendBits(const std::vector<bool>& bits, nanoseconds_t clock)
 		{
 			unsigned delta = (now - duration()) / NS_PER_TICK;
             appendInterval(delta);
+			appendPulse();
 		}
 	}
 

lib/f85/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "f85.h"
 #include "crc.h"
@@ -11,6 +12,10 @@
 #include <string.h>
 #include <algorithm>
 
+const FluxPattern SECTOR_RECORD_PATTERN(24, F85_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, F85_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
@@ -47,62 +52,49 @@ static Bytes decode(const std::vector<bool>& bits)
     return output;
 }
 
-SectorVector DurangoF85Decoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType DurangoF85Decoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-    unsigned nextSector;
-    unsigned nextTrack;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decode(rawdata);
-
-        if (bytes.size() < 4)
-            continue;
-
-        switch (bytes[0])
-        {
-            case 0xce: /* sector record */
-            {
-                headerIsValid = false;
-                nextSector = bytes[3];
-                nextTrack = bytes[1];
-
-                uint16_t wantChecksum = bytes.reader().seek(5).read_be16();
-                uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(1, 4));
-                headerIsValid = (wantChecksum == gotChecksum);
-                break;
-            }
-
-            case 0xcb: /* data record */
-            {
-                if (!headerIsValid)
-                    break;
-                if (bytes.size() < (F85_SECTOR_LENGTH + 3))
-                    continue;
-
-                const auto& payload = bytes.slice(1, F85_SECTOR_LENGTH);
-                uint16_t wantChecksum = bytes.reader().seek(F85_SECTOR_LENGTH+1).read_be16();
-                uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, payload);
-
-                int status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
 }
 
-int DurangoF85Decoder::recordMatcher(uint64_t fifo) const
+void DurangoF85Decoder::decodeSectorRecord()
 {
-    uint32_t masked = fifo & 0xffff;
-    if (masked == F85_RECORD_SEPARATOR)
-		return 6;
-    return 0;
+    /* Skip sync bits and ID byte. */
+
+    readRawBits(24);
+
+    /* Read header. */
+
+    const auto& bytes = decode(readRawBits(6*10));
+
+    _sector->logicalSector = bytes[2];
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = bytes[0];
+
+    uint16_t wantChecksum = bytes.reader().seek(4).read_be16();
+    uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(0, 4));
+    if (wantChecksum == gotChecksum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void DurangoF85Decoder::decodeDataRecord()
+{
+    /* Skip sync bits ID byte. */
+
+    readRawBits(24);
+
+    const auto& bytes = decode(readRawBits((F85_SECTOR_LENGTH+3)*10))
+        .slice(0, F85_SECTOR_LENGTH+3);
+    ByteReader br(bytes);
+
+    _sector->data = br.read(F85_SECTOR_LENGTH);
+    uint16_t wantChecksum = br.read_be16();
+    uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, _sector->data);
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/f85/f85.h

@@ -1,20 +1,21 @@
 #ifndef F85_H
 #define F85_H
 
-#define F85_RECORD_SEPARATOR 0xfffc
+#define F85_SECTOR_RECORD 0xffffce /* 1111 1111 1111 1111 1100 1110 */
+#define F85_DATA_RECORD 0xffffcb /* 1111 1111 1111 1111 1100 1101 */
 #define F85_SECTOR_LENGTH    512
 
 class Sector;
 class Fluxmap;
 
-class DurangoF85Decoder : public AbstractSoftSectorDecoder
+class DurangoF85Decoder : public AbstractDecoder
 {
 public:
     virtual ~DurangoF85Decoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
 #endif

lib/fb100/decoder.cc

@@ -0,0 +1,135 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "fb100.h"
+#include "crc.h"
+#include "bytes.h"
+#include "decoders/rawbits.h"
+#include "track.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_ID_PATTERN(16, 0xabaa);
+
+/* 
+ * Reverse engineered from a dump of the floppy drive's ROM. I have no idea how
+ * it works.
+ * 
+ * LF8BA:
+ *         clra
+ *         staa    X00B0
+ *         staa    X00B1
+ *         ldx     #$8000
+ * LF8C2:  ldaa    $00,x
+ *         inx
+ *         bsr     LF8CF
+ *         cpx     #$8011
+ *         bne     LF8C2
+ *         ldd     X00B0
+ *         rts
+ * LF8CF:
+ *         eora    X00B0
+ *         staa    X00CF
+ *         asla
+ *         asla
+ *         asla
+ *         asla
+ *         eora    X00CF
+ *         staa    X00CF
+ *         rola
+ *         rola
+ *         rola
+ *         tab
+ *         anda    #$F8
+ *         eora    X00B1
+ *         staa    X00B0
+ *         rolb
+ *         rolb
+ *         andb    #$0F
+ *         eorb    X00B0
+ *         stab    X00B0
+ *         rolb
+ *         eorb    X00CF
+ *         stab    X00B1
+ *         rts
+ */
+
+static void rol(uint8_t& b, bool& c)
+{
+    bool newc = b & 0x80;
+    b <<= 1;
+    b |= c;
+    c = newc;
+}
+
+static uint16_t checksum(const Bytes& bytes)
+{
+    uint8_t crclo = 0;
+    uint8_t crchi = 0;
+    for (uint8_t a : bytes)
+    {
+        a ^= crchi;
+        uint8_t t1 = a;
+        a <<= 4;
+        bool c = a & 0x10;
+        a ^= t1;
+        t1 = a;
+        rol(a, c);
+        rol(a, c);
+        rol(a, c);
+        uint8_t b = a;
+        a &= 0xf8;
+        a ^= crclo;
+        crchi = a;
+        rol(b, c);
+        rol(b, c);
+        b &= 0x0f;
+        b ^= crchi;
+        crchi = b;
+        rol(b, c);
+        b ^= t1;
+        crclo = b;
+    }
+
+    return (crchi << 8) | crclo;
+}
+
+AbstractDecoder::RecordType Fb100Decoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(SECTOR_ID_PATTERN, matcher);
+    if (matcher == &SECTOR_ID_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void Fb100Decoder::decodeSectorRecord()
+{
+    auto rawbits = readRawBits(FB100_RECORD_SIZE*16);
+
+    const Bytes bytes = decodeFmMfm(rawbits).slice(0, FB100_RECORD_SIZE);
+    ByteReader br(bytes);
+    br.seek(1);
+    const Bytes id = br.read(FB100_ID_SIZE);
+    uint16_t wantIdCrc = br.read_be16();
+    uint16_t gotIdCrc = checksum(id);
+    const Bytes payload = br.read(FB100_PAYLOAD_SIZE);
+    uint16_t wantPayloadCrc = br.read_be16();
+    uint16_t gotPayloadCrc = checksum(payload);
+
+    if (wantIdCrc != gotIdCrc)
+        return;
+
+    uint8_t abssector = id[2];
+    _sector->logicalTrack = abssector >> 1;
+    _sector->logicalSide = 0;
+    _sector->logicalSector = abssector & 1;
+    _sector->data.writer().append(id.slice(5, 12)).append(payload);
+
+    _sector->status = (wantPayloadCrc == gotPayloadCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

lib/fb100/fb100.h

@@ -0,0 +1,22 @@
+#ifndef FB100_H
+#define FB100_H
+
+#define FB100_RECORD_SIZE 0x516 /* bytes */
+#define FB100_ID_SIZE 17
+#define FB100_PAYLOAD_SIZE 0x500
+
+class Sector;
+class Fluxmap;
+class Track;
+
+class Fb100Decoder : public AbstractDecoder
+{
+public:
+    virtual ~Fb100Decoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+};
+
+#endif
+

lib/flags.cc

@@ -1,25 +1,71 @@
 #include "globals.h"
 #include "flags.h"
 
+static FlagGroup* currentFlagGroup;
 static std::vector<Flag*> all_flags;
 static std::map<const std::string, Flag*> flags_by_name;
 
-Flag::Flag(const std::vector<std::string>& names, const std::string helptext):
-    _names(names),
-    _helptext(helptext)
-{
-    for (auto& name : names)
-    {
-        if (flags_by_name.find(name) != flags_by_name.end())
-            Error() << "two flags use the name '" << name << "'";
-        flags_by_name[name] = this;
-    }
+static void doHelp();
 
-    all_flags.push_back(this);
+static FlagGroup helpGroup;
+static ActionFlag helpFlag = ActionFlag(
+    { "--help", "-h" },
+    "Shows the help.",
+    doHelp);
+
+FlagGroup::FlagGroup(const std::initializer_list<FlagGroup*> groups):
+    _groups(groups.begin(), groups.end())
+{
+    currentFlagGroup = this;
 }
 
-void Flag::parseFlags(int argc, const char* argv[])
+FlagGroup::FlagGroup()
 {
+    currentFlagGroup = this;
+}
+
+void FlagGroup::addFlag(Flag* flag)
+{
+    _flags.push_back(flag);
+}
+
+void FlagGroup::parseFlags(int argc, const char* argv[])
+{
+    if (_initialised)
+        throw std::runtime_error("called parseFlags() twice");
+
+    /* Recursively accumulate a list of all flags. */
+
+    all_flags.clear();
+    flags_by_name.clear();
+    std::function<void(FlagGroup*)> recurse;
+    recurse = [&](FlagGroup* group)
+    {
+        if (group->_initialised)
+            return;
+        
+        for (FlagGroup* subgroup : group->_groups)
+            recurse(subgroup);
+
+        for (Flag* flag : group->_flags)
+        {
+            for (const auto& name : flag->names())
+            {
+                if (flags_by_name.find(name) != flags_by_name.end())
+                    Error() << "two flags use the name '" << name << "'";
+                flags_by_name[name] = flag;
+            }
+
+            all_flags.push_back(flag);
+        }
+
+        group->_initialised = true;
+    };
+    recurse(this);
+    recurse(&helpGroup);
+
+    /* Now actually parse them. */
+
     int index = 1;
     while (index < argc)
     {
@@ -76,15 +122,28 @@ void Flag::parseFlags(int argc, const char* argv[])
         if (usesthat && flag->second->hasArgument())
             index++;
     }
-        
+}
+
+void FlagGroup::checkInitialised() const
+{
+    if (!_initialised)
+        throw std::runtime_error("Attempt to access uninitialised flag");
+}
+
+Flag::Flag(const std::vector<std::string>& names, const std::string helptext):
+    _group(*currentFlagGroup),
+    _names(names),
+    _helptext(helptext)
+{
+    _group.addFlag(this);
 }
 
 void BoolFlag::set(const std::string& value)
 {
 	if ((value == "true") || (value == "y"))
-		this->value = true;
+		_value = true;
 	else if ((value == "false") || (value == "n"))
-		this->value = false;
+		_value = false;
 	else
 		Error() << "can't parse '" << value << "'; try 'true' or 'false'";
 }
@@ -110,8 +169,3 @@ static void doHelp()
     }
     exit(0);
 }
-
-static ActionFlag helpFlag = ActionFlag(
-    { "--help", "-h" },
-    "Shows the help.",
-    doHelp);

lib/flags.h

@@ -2,15 +2,36 @@
 #define FLAGS_H
 
 class DataSpec;
+class Flag;
+
+class FlagGroup
+{
+private:
+    FlagGroup(const FlagGroup& group);
+public:
+    FlagGroup(const std::initializer_list<FlagGroup*> groups);
+    FlagGroup();
+
+public:
+    void parseFlags(int argc, const char* argv[]);
+    void addFlag(Flag* flag);
+    void checkInitialised() const;
+
+private:
+    bool _initialised = false;
+    const std::vector<FlagGroup*> _groups;
+    std::vector<Flag*> _flags;
+};
 
 class Flag
 {
 public:
-    static void parseFlags(int argc, const char* argv[]);
-
     Flag(const std::vector<std::string>& names, const std::string helptext);
     virtual ~Flag() {};
 
+    void checkInitialised() const
+    { _group.checkInitialised(); }
+
     const std::string& name() const { return _names[0]; }
     const std::vector<std::string> names() const { return _names; }
     const std::string& helptext() const { return _helptext; }
@@ -20,6 +41,7 @@ public:
     virtual void set(const std::string& value) = 0;
 
 private:
+    FlagGroup& _group;
     const std::vector<std::string> _names;
     const std::string _helptext;
 };
@@ -48,7 +70,8 @@ public:
         Flag(names, helptext)
     {}
 
-    operator bool() const { return _value; }
+    operator bool() const
+    { checkInitialised(); return _value; }
 
     bool hasArgument() const { return false; }
     const std::string defaultValueAsString() const { return "false"; }
@@ -65,16 +88,26 @@ public:
     ValueFlag(const std::vector<std::string>& names, const std::string helptext,
             const T defaultValue):
         Flag(names, helptext),
-        defaultValue(defaultValue),
-        value(defaultValue)
+        _defaultValue(defaultValue),
+        _value(defaultValue)
     {}
 
-    operator T() const { return value; }
+    const T& get() const
+    { checkInitialised(); return _value; }
+
+    operator const T& () const 
+    { return get(); }
+
+    void setDefaultValue(T value)
+    {
+        _value = _defaultValue = value;
+    }
 
     bool hasArgument() const { return true; }
 
-    T defaultValue;
-    T value;
+protected:
+    T _defaultValue;
+    T _value;
 };
 
 class StringFlag : public ValueFlag<std::string>
@@ -85,8 +118,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return defaultValue; }
-    void set(const std::string& value) { this->value = value; }
+    const std::string defaultValueAsString() const { return _defaultValue; }
+    void set(const std::string& value) { _value = value; }
 };
 
 class IntFlag : public ValueFlag<int>
@@ -97,8 +130,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return std::to_string(defaultValue); }
-    void set(const std::string& value) { this->value = std::stoi(value); }
+    const std::string defaultValueAsString() const { return std::to_string(_defaultValue); }
+    void set(const std::string& value) { _value = std::stoi(value); }
 };
 
 class DoubleFlag : public ValueFlag<double>
@@ -109,8 +142,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return std::to_string(defaultValue); }
-    void set(const std::string& value) { this->value = std::stod(value); }
+    const std::string defaultValueAsString() const { return std::to_string(_defaultValue); }
+    void set(const std::string& value) { _value = std::stod(value); }
 };
 
 class BoolFlag : public ValueFlag<double>
@@ -121,7 +154,7 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return defaultValue ? "true" : "false"; }
+    const std::string defaultValueAsString() const { return _defaultValue ? "true" : "false"; }
     void set(const std::string& value);
 };
 

lib/fluxmap.cc

@@ -32,6 +32,11 @@ Fluxmap& Fluxmap::appendInterval(uint32_t ticks)
         ticks -= 0x7f;
     }
     appendByte((uint8_t)ticks);
+    return *this;
+}
+
+Fluxmap& Fluxmap::appendPulse()
+{
     appendByte(0x80);
     return *this;
 }
@@ -72,33 +77,3 @@ void Fluxmap::precompensate(int threshold_ticks, int amount_ticks)
         }
     }
 }
-
-int FluxmapReader::read(unsigned& ticks)
-{
-    ticks = 0;
-
-    while (_cursor < _size)
-    {
-        uint8_t b = _bytes[_cursor++];
-        if (b < 0x80)
-            ticks += b;
-        else
-            return b;
-    }
-
-    return -1;
-}
-
-int FluxmapReader::readPulse(unsigned& ticks)
-{
-    ticks = 0;
-
-    for (;;)
-    {
-        unsigned thisTicks;
-        int opcode = read(thisTicks);
-        ticks += thisTicks;
-        if ((opcode == -1) || (opcode == 0x80))
-            return opcode;
-    }
-}

lib/fluxmap.h

@@ -2,11 +2,23 @@
 #define FLUXMAP_H
 
 #include "bytes.h"
+#include "protocol.h"
 
 class RawBits;
 
 class Fluxmap
 {
+public:
+    struct Position
+    {
+        unsigned bytes = 0;
+        unsigned ticks = 0;
+        unsigned zeroes = 0;
+
+        nanoseconds_t ns() const
+        { return ticks * NS_PER_TICK; }
+    };
+
 public:
     nanoseconds_t duration() const { return _duration; }
     size_t bytes() const { return _bytes.size(); }
@@ -20,6 +32,7 @@ public:
 	}
 
     Fluxmap& appendInterval(uint32_t ticks);
+    Fluxmap& appendPulse();
     Fluxmap& appendIndex();
 
     Fluxmap& appendBytes(const Bytes& bytes);
@@ -30,9 +43,6 @@ public:
         return appendBytes(&byte, 1);
     }
 
-    nanoseconds_t guessClock() const;
-	const RawBits decodeToBits(nanoseconds_t clock_period) const;
-
 	Fluxmap& appendBits(const std::vector<bool>& bits, nanoseconds_t clock);
 
 	void precompensate(int threshold_ticks, int amount_ticks);
@@ -43,33 +53,4 @@ private:
     Bytes _bytes;
 };
 
-class FluxmapReader
-{
-public:
-    FluxmapReader(const Fluxmap& fluxmap):
-        _fluxmap(fluxmap),
-        _bytes(fluxmap.ptr()),
-        _size(fluxmap.bytes())
-    {
-        rewind();
-    }
-
-    FluxmapReader(const Fluxmap&& fluxmap) = delete;
-
-    void rewind()
-    { _cursor = 0; }
-
-    size_t tell() const
-    { return _cursor; }
-
-    int read(unsigned& ticks);
-    int readPulse(unsigned& ticks);
-
-private:
-    const Fluxmap& _fluxmap;
-    const uint8_t* _bytes;
-    const size_t _size;
-    size_t _cursor;
-};
-
 #endif

lib/fluxreader/fluxreader.h

@@ -1,29 +0,0 @@
-#ifndef FLUXREADER_H
-#define FLUXREADER_H
-
-class Fluxmap;
-class DataSpec;
-
-class FluxReader
-{
-public:
-    virtual ~FluxReader() {}
-
-private:
-    static std::unique_ptr<FluxReader> createSqliteFluxReader(const std::string& filename);
-    static std::unique_ptr<FluxReader> createHardwareFluxReader(unsigned drive);
-    static std::unique_ptr<FluxReader> createStreamFluxReader(const std::string& path);
-
-public:
-    static std::unique_ptr<FluxReader> create(const DataSpec& spec);
-
-public:
-    virtual std::unique_ptr<Fluxmap> readFlux(int track, int side) = 0;
-    virtual void recalibrate() {}
-    virtual bool retryable() { return false; }
-};
-
-extern void setHardwareFluxReaderRevolutions(int revolutions);
-
-#endif
-

lib/fluxreader/kryoflux.cc

@@ -1,121 +0,0 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "kryoflux.h"
-#include "protocol.h"
-#include "fmt/format.h"
-#include <fstream>
-#include <glob.h>
-
-#define SCLK_HZ 24027428.57142857
-#define TICKS_PER_SCLK (TICK_FREQUENCY / SCLK_HZ)
-
-std::unique_ptr<Fluxmap> readStream(const std::string& dir, unsigned track, unsigned side)
-{
-    std::string suffix = fmt::format("{:02}.{}.raw", track, side);
-    std::string pattern = fmt::format("{}*{}", dir, suffix);
-    glob_t globdata;
-    if (glob(pattern.c_str(), GLOB_NOSORT, NULL, &globdata))
-        Error() << fmt::format("cannot access path '{}'", dir);
-    if (globdata.gl_pathc != 1)
-        Error() << fmt::format("data is ambiguous --- multiple files end in {}", suffix);
-    std::string filename = globdata.gl_pathv[0];
-    globfree(&globdata);
-
-    return readStream(filename);
-}
-
-std::unique_ptr<Fluxmap> readStream(const std::string& filename)
-{
-    std::ifstream f(filename, std::ios::in | std::ios::binary);
-    if (!f.is_open())
-		Error() << fmt::format("cannot open input file '{}'", filename);
-
-    return readStream(f);
-}
-
-std::unique_ptr<Fluxmap> readStream(std::istream& f)
-{
-    std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
-    auto writeFlux = [&](uint32_t sclk)
-    {
-        int ticks = (double)sclk * TICKS_PER_SCLK;
-        fluxmap->appendInterval(ticks);
-    };
-
-    uint32_t extrasclks = 0;
-    for (;;)
-    {
-        int b = f.get(); /* returns -1 or UNSIGNED char */
-        if (b == -1)
-            break;
-
-        switch (b)
-        {
-            case 0x0d: /* OOB block */
-            {
-                int blocktype = f.get();
-                (void) blocktype;
-                uint16_t blocklen = f.get() | (f.get()<<8);
-                if (f.fail() || f.eof())
-                    goto finished;
-
-                while (blocklen--)
-                    f.get();
-                break;
-            }
-
-            default:
-            {
-                if ((b >= 0x00) && (b <= 0x07))
-                {
-                    /* Flux2: double byte value */
-                    b = (b<<8) | f.get();
-                    writeFlux(extrasclks + b);
-                    extrasclks = 0;
-                }
-                else if (b == 0x08)
-                {
-                    /* Nop1: do nothing */
-                }
-                else if (b == 0x09)
-                {
-                    /* Nop2: skip one byte */
-                    f.get();
-                }
-                else if (b == 0x0a)
-                {
-                    /* Nop3: skip two bytes */
-                    f.get();
-                    f.get();
-                }
-                else if (b == 0x0b)
-                {
-                    /* Ovl16: the next block is 0x10000 sclks longer than normal. */
-                    extrasclks += 0x10000;
-                }
-                else if (b == 0x0c)
-                {
-                    /* Flux3: triple byte value */
-                    int ticks = f.get() << 8;
-                    ticks |= f.get();
-                    writeFlux(extrasclks + ticks);
-                    extrasclks = 0;
-                }
-                else if ((b >= 0x0e) && (b <= 0xff))
-                {
-                    /* Flux1: single byte value */
-                    writeFlux(extrasclks + b);
-                    extrasclks = 0;
-                }
-                else
-                    Error() << fmt::format(
-                        "unknown stream block byte 0x{:02x} at 0x{:08x}", b, (uint64_t)f.tellg()-1);
-            }
-        }
-    }
-
-finished:
-    if (!f.eof())
-        Error() << "I/O error reading stream";
-    return fluxmap;
-}

lib/fluxsink/fluxsink.cc

@@ -0,0 +1,24 @@
+#include "globals.h"
+#include "flags.h"
+#include "dataspec.h"
+#include "fluxsink/fluxsink.h"
+
+static bool ends_with(const std::string& value, const std::string& ending)
+{
+    if (ending.size() > value.size())
+        return false;
+    return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
+}
+
+std::unique_ptr<FluxSink> FluxSink::create(const DataSpec& spec)
+{
+    const auto& filename = spec.filename;
+
+    if (filename.empty())
+        return createHardwareFluxSink(spec.drive);
+    else if (ends_with(filename, ".flux"))
+        return createSqliteFluxSink(filename);
+
+    Error() << "unrecognised flux filename extension";
+    return std::unique_ptr<FluxSink>();
+}

lib/fluxsink/fluxsink.h

@@ -0,0 +1,26 @@
+#ifndef FLUXSINK_H
+#define FLUXSINK_H
+
+class Fluxmap;
+class DataSpec;
+
+class FluxSink
+{
+public:
+    virtual ~FluxSink() {}
+
+private:
+    static std::unique_ptr<FluxSink> createSqliteFluxSink(const std::string& filename);
+    static std::unique_ptr<FluxSink> createHardwareFluxSink(unsigned drive);
+
+public:
+    static std::unique_ptr<FluxSink> create(const DataSpec& spec);
+
+public:
+    virtual void writeFlux(int track, int side, Fluxmap& fluxmap) = 0;
+};
+
+extern void setHardwareFluxSinkDensity(bool high_density);
+
+#endif
+

lib/fluxsink/hardwarefluxsink.cc

@@ -0,0 +1,46 @@
+#include "globals.h"
+#include "flags.h"
+#include "fluxmap.h"
+#include "usb.h"
+#include "fluxsink/fluxsink.h"
+
+static bool high_density = false;
+
+void setHardwareFluxSinkDensity(bool high_density)
+{
+	::high_density = high_density;
+}
+
+class HardwareFluxSink : public FluxSink
+{
+public:
+    HardwareFluxSink(unsigned drive):
+        _drive(drive)
+    {
+    }
+
+    ~HardwareFluxSink()
+    {
+    }
+
+public:
+    void writeFlux(int track, int side, Fluxmap& fluxmap)
+    {
+        usbSetDrive(_drive, high_density);
+        usbSeek(track);
+
+        Bytes crunched = fluxmap.rawBytes().crunch();
+        return usbWrite(side, crunched);
+    }
+
+private:
+    unsigned _drive;
+};
+
+std::unique_ptr<FluxSink> FluxSink::createHardwareFluxSink(unsigned drive)
+{
+    return std::unique_ptr<FluxSink>(new HardwareFluxSink(drive));
+}
+
+
+

lib/fluxsink/sqlitefluxsink.cc

@@ -0,0 +1,47 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "sql.h"
+#include "fluxsink/fluxsink.h"
+#include "fmt/format.h"
+
+class SqliteFluxSink : public FluxSink
+{
+public:
+    SqliteFluxSink(const std::string& filename)
+    {
+		_outdb = sqlOpen(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
+		int oldVersion = sqlReadIntProperty(_outdb, "version");
+		if ((oldVersion != 0) && (oldVersion != FLUX_VERSION_CURRENT))
+            Error() << fmt::format("that flux file is version {}, but this client is for version {}",
+                oldVersion, FLUX_VERSION_CURRENT);
+
+		sqlPrepareFlux(_outdb);
+		sqlStmt(_outdb, "BEGIN;");
+        sqlWriteIntProperty(_outdb, "version", FLUX_VERSION_CURRENT);
+    }
+
+    ~SqliteFluxSink()
+    {
+        if (_outdb)
+		{
+			sqlStmt(_outdb, "COMMIT;");
+            sqlClose(_outdb);
+		}
+    }
+
+public:
+    void writeFlux(int track, int side, Fluxmap& fluxmap)
+    {
+        return sqlWriteFlux(_outdb, track, side, fluxmap);
+    }
+
+private:
+    sqlite3* _outdb;
+};
+
+std::unique_ptr<FluxSink> FluxSink::createSqliteFluxSink(const std::string& filename)
+{
+    return std::unique_ptr<FluxSink>(new SqliteFluxSink(filename));
+}
+
+

lib/fluxsource/fluxsource.cc

@@ -1,7 +1,7 @@
 #include "globals.h"
 #include "flags.h"
 #include "dataspec.h"
-#include "fluxreader.h"
+#include "fluxsource/fluxsource.h"
 
 static bool ends_with(const std::string& value, const std::string& ending)
 {
@@ -10,17 +10,17 @@ static bool ends_with(const std::string& value, const std::string& ending)
     return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
 }
 
-std::unique_ptr<FluxReader> FluxReader::create(const DataSpec& spec)
+std::unique_ptr<FluxSource> FluxSource::create(const DataSpec& spec)
 {
     const auto& filename = spec.filename;
 
     if (filename.empty())
-        return createHardwareFluxReader(spec.drive);
+        return createHardwareFluxSource(spec.drive);
     else if (ends_with(filename, ".flux"))
-        return createSqliteFluxReader(filename);
+        return createSqliteFluxSource(filename);
     else if (ends_with(filename, "/"))
-        return createStreamFluxReader(filename);
+        return createStreamFluxSource(filename);
 
     Error() << "unrecognised flux filename extension";
-    return std::unique_ptr<FluxReader>();
+    return std::unique_ptr<FluxSource>();
 }

lib/fluxsource/fluxsource.h

@@ -0,0 +1,34 @@
+#ifndef FLUXSOURCE_H
+#define FLUXSOURCE_H
+
+#include "flags.h"
+
+extern FlagGroup hardwareFluxSourceFlags;
+
+class Fluxmap;
+class DataSpec;
+
+class FluxSource
+{
+public:
+    virtual ~FluxSource() {}
+
+private:
+    static std::unique_ptr<FluxSource> createSqliteFluxSource(const std::string& filename);
+    static std::unique_ptr<FluxSource> createHardwareFluxSource(unsigned drive);
+    static std::unique_ptr<FluxSource> createStreamFluxSource(const std::string& path);
+
+public:
+    static std::unique_ptr<FluxSource> create(const DataSpec& spec);
+
+public:
+    virtual std::unique_ptr<Fluxmap> readFlux(int track, int side) = 0;
+    virtual void recalibrate() {}
+    virtual bool retryable() { return false; }
+};
+
+extern void setHardwareFluxSourceRevolutions(int revolutions);
+extern void setHardwareFluxSourceDensity(bool high_density);
+
+#endif
+

lib/fluxsource/hardwarefluxsource.cc

@@ -0,0 +1,70 @@
+#include "globals.h"
+#include "flags.h"
+#include "fluxmap.h"
+#include "usb.h"
+#include "fluxsource/fluxsource.h"
+
+FlagGroup hardwareFluxSourceFlags;
+
+static IntFlag revolutions(
+    { "--revolutions" },
+    "read this many revolutions of the disk",
+    1);
+
+static bool high_density = false;
+
+void setHardwareFluxSourceDensity(bool high_density)
+{
+	::high_density = high_density;
+}
+
+class HardwareFluxSource : public FluxSource
+{
+public:
+    HardwareFluxSource(unsigned drive):
+        _drive(drive)
+    {
+    }
+
+    ~HardwareFluxSource()
+    {
+    }
+
+public:
+    std::unique_ptr<Fluxmap> readFlux(int track, int side)
+    {
+        usbSetDrive(_drive, high_density);
+        usbSeek(track);
+        Bytes crunched = usbRead(side, revolutions);
+        auto fluxmap = std::make_unique<Fluxmap>();
+        fluxmap->appendBytes(crunched.uncrunch());
+        return fluxmap;
+    }
+
+    void recalibrate()
+    {
+        usbRecalibrate();
+    }
+
+    bool retryable()
+    {
+        return true;
+    }
+
+private:
+    unsigned _drive;
+    unsigned _revolutions;
+};
+
+void setHardwareFluxSourceRevolutions(int revolutions)
+{
+    ::revolutions.setDefaultValue(revolutions);
+}
+
+std::unique_ptr<FluxSource> FluxSource::createHardwareFluxSource(unsigned drive)
+{
+    return std::unique_ptr<FluxSource>(new HardwareFluxSource(drive));
+}
+
+
+

lib/fluxsource/kryoflux.cc

@@ -0,0 +1,247 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "kryoflux.h"
+#include "protocol.h"
+#include "fmt/format.h"
+#include <fstream>
+#include <sys/types.h>
+#include <dirent.h>
+
+#define MCLK_HZ (((18432000.0 * 73.0) / 14.0) / 2.0)
+#define SCLK_HZ (MCLK_HZ / 2)
+#define ICLK_HZ (MCLK_HZ / 16)
+
+#define TICKS_PER_SCLK (TICK_FREQUENCY / SCLK_HZ)
+
+static bool has_suffix(const std::string& haystack, const std::string& needle)
+{
+    if (needle.length() > haystack.length())
+        return false;
+    
+    return haystack.compare(haystack.length() - needle.length(), needle.length(), needle) == 0;
+}
+
+std::unique_ptr<Fluxmap> readStream(const std::string& dir, unsigned track, unsigned side)
+{
+    std::string suffix = fmt::format("{:02}.{}.raw", track, side);
+
+    DIR* dirp = opendir(dir.c_str());
+    if (!dirp)
+        Error() << fmt::format("cannot access path '{}'", dir);
+
+    std::string filename;
+    for (;;)
+    {
+        struct dirent* de = readdir(dirp);
+        if (!de)
+            break;
+        
+        if (has_suffix(de->d_name, suffix))
+        {
+            if (!filename.empty())
+                Error() << fmt::format("data is ambiguous --- multiple files end in {}", suffix);
+            filename = fmt::format("{}/{}", dir, de->d_name);
+        }
+    }
+    closedir(dirp);
+
+    if (filename.empty())
+        Error() << fmt::format("failed to find track {} side {} in {}", track, side, dir);
+
+    return readStream(filename);
+}
+
+std::unique_ptr<Fluxmap> readStream(const std::string& filename)
+{
+    std::ifstream f(filename, std::ios::in | std::ios::binary);
+    if (!f.is_open())
+		Error() << fmt::format("cannot open input file '{}'", filename);
+
+    Bytes bytes;
+    ByteWriter bw(bytes);
+    bw.append(f);
+
+    return readStream(bytes);
+}
+
+std::unique_ptr<Fluxmap> readStream(const Bytes& bytes)
+{
+    ByteReader br(bytes);
+
+    /* Pass 1: scan the stream looking for index marks. */
+    
+    std::set<uint32_t> indexmarks;
+    br.seek(0);
+    while (!br.eof())
+    {
+        uint8_t b = br.read_8();
+        unsigned len = 0;
+        switch (b)
+        {
+            case 0x0d: /* OOB block */
+            {
+                int blocktype = br.read_8();
+                len = br.read_le16();
+                if (br.eof())
+                    goto finished_pass_1;
+
+                if (blocktype == 0x02)
+                {
+                    /* index data, sent asynchronously */
+                    uint32_t streampos = br.read_le32();
+                    indexmarks.insert(streampos);
+                    len -= 4;
+                }
+                break;
+            }
+
+            default:
+            {
+                if ((b >= 0x00) && (b <= 0x07))
+                {
+                    /* Flux2: double byte value */
+                    len = 1;
+                }
+                else if (b == 0x08)
+                {
+                    /* Nop1: do nothing */
+                    len = 0;
+                }
+                else if (b == 0x09)
+                {
+                    /* Nop2: skip one byte */
+                    len = 1;
+                }
+                else if (b == 0x0a)
+                {
+                    /* Nop3: skip two bytes */
+                    len = 2;
+                }
+                else if (b == 0x0b)
+                {
+                    /* Ovl16: the next block is 0x10000 sclks longer than normal. */
+                    len = 0;
+                }
+                else if (b == 0x0c)
+                {
+                    /* Flux3: triple byte value */
+                    len = 2;
+                }
+                else if ((b >= 0x0e) && (b <= 0xff))
+                {
+                    /* Flux1: single byte value */
+                    len = 0;
+                }
+                else
+                    Error() << fmt::format(
+                        "unknown stream block byte 0x{:02x} at 0x{:08x}", b, (uint64_t)br.pos-1);
+            }
+        }
+
+        br.skip(len);
+    }
+finished_pass_1:
+
+    /* Pass 2: actually read the data. */
+
+    std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+    int streamdelta = 0;
+    auto writeFlux = [&](uint32_t sclk)
+    {
+        const auto& nextindex = indexmarks.begin();
+        if (nextindex != indexmarks.end())
+        {
+            uint32_t nextindexpos = *nextindex + streamdelta;
+            if (br.pos >= nextindexpos)
+            {
+                fluxmap->appendIndex();
+                indexmarks.erase(nextindex);
+            }
+        }
+        int ticks = (double)sclk * TICKS_PER_SCLK;
+        fluxmap->appendInterval(ticks);
+        fluxmap->appendPulse();
+    };
+
+    uint32_t extrasclks = 0;
+    br.seek(0);
+    while (!br.eof())
+    {
+        unsigned b = br.read_8();
+        switch (b)
+        {
+            case 0x0d: /* OOB block */
+            {
+                int blocktype = br.read_8();
+                uint16_t blocklen = br.read_le16();
+                if (br.eof())
+                    goto finished_pass_2;
+
+                switch (blocktype)
+                {
+                    case 0x01: /* streaminfo */
+                    {
+                        uint32_t blockpos = br.pos - 3;
+                        streamdelta = blockpos - br.read_le32();
+                        blocklen -= 4;
+                        break;
+                    }
+                }
+
+                br.skip(blocklen);
+                break;
+            }
+
+            default:
+            {
+                if ((b >= 0x00) && (b <= 0x07))
+                {
+                    /* Flux2: double byte value */
+                    b = (b<<8) | br.read_8();
+                    writeFlux(extrasclks + b);
+                    extrasclks = 0;
+                }
+                else if (b == 0x08)
+                {
+                    /* Nop1: do nothing */
+                }
+                else if (b == 0x09)
+                {
+                    /* Nop2: skip one byte */
+                    br.skip(1);
+                }
+                else if (b == 0x0a)
+                {
+                    /* Nop3: skip two bytes */
+                    br.skip(2);
+                }
+                else if (b == 0x0b)
+                {
+                    /* Ovl16: the next block is 0x10000 sclks longer than normal. */
+                    extrasclks += 0x10000;
+                }
+                else if (b == 0x0c)
+                {
+                    /* Flux3: triple byte value */
+                    int ticks = br.read_be16(); /* yes, really big-endian */
+                    writeFlux(extrasclks + ticks);
+                    extrasclks = 0;
+                }
+                else if ((b >= 0x0e) && (b <= 0xff))
+                {
+                    /* Flux1: single byte value */
+                    writeFlux(extrasclks + b);
+                    extrasclks = 0;
+                }
+                else
+                    Error() << fmt::format(
+                        "unknown stream block byte 0x{:02x} at 0x{:08x}", b, (uint64_t)br.pos-1);
+            }
+        }
+    }
+
+finished_pass_2:
+    if (!br.eof())
+        Error() << "I/O error reading stream";
+    return fluxmap;
+}

lib/fluxsource/kryoflux.h

@@ -3,6 +3,6 @@
 
 extern std::unique_ptr<Fluxmap> readStream(const std::string& dir, unsigned track, unsigned side);
 extern std::unique_ptr<Fluxmap> readStream(const std::string& path);
-extern std::unique_ptr<Fluxmap> readStream(std::istream& stream);
+extern std::unique_ptr<Fluxmap> readStream(const Bytes& bytes);
 
 #endif

lib/fluxsource/sqlitefluxsource.cc

@@ -1,13 +1,13 @@
 #include "globals.h"
 #include "fluxmap.h"
 #include "sql.h"
-#include "fluxreader.h"
+#include "fluxsource/fluxsource.h"
 #include "fmt/format.h"
 
-class SqliteFluxReader : public FluxReader
+class SqliteFluxSource : public FluxSource
 {
 public:
-    SqliteFluxReader(const std::string& filename)
+    SqliteFluxSource(const std::string& filename)
     {
         _indb = sqlOpen(filename, SQLITE_OPEN_READONLY);
         int version = sqlGetVersion(_indb);
@@ -16,7 +16,7 @@ public:
                 version, FLUX_VERSION_CURRENT);
     }
 
-    ~SqliteFluxReader()
+    ~SqliteFluxSource()
     {
         if (_indb)
             sqlClose(_indb);
@@ -34,9 +34,9 @@ private:
     sqlite3* _indb;
 };
 
-std::unique_ptr<FluxReader> FluxReader::createSqliteFluxReader(const std::string& filename)
+std::unique_ptr<FluxSource> FluxSource::createSqliteFluxSource(const std::string& filename)
 {
-    return std::unique_ptr<FluxReader>(new SqliteFluxReader(filename));
+    return std::unique_ptr<FluxSource>(new SqliteFluxSource(filename));
 }
 
 

lib/fluxsource/streamfluxsource.cc

@@ -1,17 +1,17 @@
 #include "globals.h"
 #include "fluxmap.h"
 #include "kryoflux.h"
-#include "fluxreader.h"
+#include "fluxsource/fluxsource.h"
 
-class StreamFluxReader : public FluxReader
+class StreamFluxSource : public FluxSource
 {
 public:
-    StreamFluxReader(const std::string& path):
+    StreamFluxSource(const std::string& path):
         _path(path)
     {
     }
 
-    ~StreamFluxReader()
+    ~StreamFluxSource()
     {
     }
 
@@ -27,7 +27,7 @@ private:
     const std::string& _path;
 };
 
-std::unique_ptr<FluxReader> FluxReader::createStreamFluxReader(const std::string& path)
+std::unique_ptr<FluxSource> FluxSource::createStreamFluxSource(const std::string& path)
 {
-    return std::unique_ptr<FluxReader>(new StreamFluxReader(path));
+    return std::unique_ptr<FluxSource>(new StreamFluxSource(path));
 }

lib/ibm/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "ibm.h"
 #include "crc.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "sector.h"
 #include "record.h"
 #include <string.h>
@@ -10,159 +11,143 @@
 static_assert(std::is_trivially_copyable<IbmIdam>::value,
 		"IbmIdam is not trivially copyable");
 
-SectorVector AbstractIbmDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    bool idamValid = false;
-    IbmIdam idam;
-    std::vector<std::unique_ptr<Sector>> sectors;
+/*
+ * The markers at the beginning of records are special, and have
+ * missing clock pulses, allowing them to be found by the logic.
+ * 
+ * IAM record:
+ * flux:   XXXX-XXX-XXXX-X- = 0xf77a
+ * clock:  X X - X - X X X  = 0xd7
+ * data:    X X X X X X - - = 0xfc
+ * 
+ * (We just ignore this one --- it's useless and optional.)
+ */
 
-    for (auto& rawrecord : rawRecords)
+/* 
+ * IDAM record:
+ * flux:   XXXX-X-X-XXXXXX- = 0xf57e
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X X X - = 0xfe
+ */
+const FluxPattern FM_IDAM_PATTERN(16, 0xf57e);
+
+/* 
+ * DAM1 record:
+ * flux:   XXXX-X-X-XX-X-X- = 0xf56a
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - - - = 0xf8
+ */
+const FluxPattern FM_DAM1_PATTERN(16, 0xf56a);
+
+/* 
+ * DAM2 record:
+ * flux:   XXXX-X-X-XX-XXXX = 0xf56f
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - X X = 0xfb
+ */
+const FluxPattern FM_DAM2_PATTERN(16, 0xf56f);
+
+/* 
+ * TRS80DAM1 record:
+ * flux:   XXXX-X-X-XX-X-XX = 0xf56b
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - - X = 0xf9
+ */
+const FluxPattern FM_TRS80DAM1_PATTERN(16, 0xf56b);
+
+/* 
+ * TRS80DAM2 record:
+ * flux:   XXXX-X-X-XX-XXX- = 0xf56c
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - X - = 0xfa
+ */
+const FluxPattern FM_TRS80DAM2_PATTERN(16, 0xf56c);
+
+/* MFM record separator:
+ * 0xA1 is:
+ * data:    1  0  1  0  0  0  0  1  = 0xa1
+ * mfm:     01 00 01 00 10 10 10 01 = 0x44a9
+ * special: 01 00 01 00 10 00 10 01 = 0x4489
+ *                       ^^^^^
+ * When shifted out of phase, the special 0xa1 byte becomes an illegal
+ * encoding (you can't do 10 00). So this can't be spoofed by user data.
+ * 
+ * shifted: 10 00 10 01 00 01 00 1
+ */
+const FluxPattern MFM_PATTERN(16, 0x4489);
+
+const FluxMatchers ANY_RECORD_PATTERN(
     {
-        const Bytes bytes = decodeFmMfm(rawrecord->data);
-        int headerSize = skipHeaderBytes();
-        Bytes data = bytes.slice(headerSize, bytes.size() - headerSize);
-
-        switch (data[0])
-        {
-            case IBM_IAM:
-                /* Track header. Ignore. */
-                break;
-
-            case IBM_IDAM:
-            {
-                if (data.size() < sizeof(idam))
-                    break;
-                memcpy(&idam, data.cbegin(), sizeof(idam));
-
-				uint16_t crc = crc16(CCITT_POLY, bytes.slice(0, offsetof(IbmIdam, crc) + headerSize));
-				uint16_t wantedCrc = (idam.crc[0]<<8) | idam.crc[1];
-				idamValid = (crc == wantedCrc);
-                break;
-            }
-
-            case IBM_DAM1:
-            case IBM_DAM2:
-            case IBM_TRS80DAM1:
-            case IBM_TRS80DAM2:
-            {
-                if (!idamValid)
-                    break;
-                
-                unsigned size = 1 << (idam.sectorSize + 7);
-                data.resize(IBM_DAM_LEN + size + 2);
-
-				uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, IBM_DAM_LEN+size+headerSize));
-				uint16_t wantedCrc = data.reader().seek(IBM_DAM_LEN+size).read_be16();
-				int status = (gotCrc == wantedCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                int sectorNum = idam.sector - _sectorIdBase;
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, idam.cylinder, idam.side, sectorNum,
-                        data.slice(IBM_DAM_LEN, size)));
-                sectors.push_back(std::move(sector));
-                idamValid = false;
-                break;
-            }
-        }
+        &MFM_PATTERN,
+        &FM_IDAM_PATTERN,
+        &FM_DAM1_PATTERN,
+        &FM_DAM2_PATTERN,
+        &FM_TRS80DAM1_PATTERN,
+        &FM_TRS80DAM2_PATTERN,
     }
+);
 
-    return sectors;
-}
-
-nanoseconds_t IbmMfmDecoder::guessClock(Fluxmap& fluxmap) const
+AbstractDecoder::RecordType IbmDecoder::advanceToNextRecord()
 {
-    return fluxmap.guessClock() / 2;
-}
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
 
-int IbmFmDecoder::recordMatcher(uint64_t fifo) const
-{
-    /*
-     * The markers at the beginning of records are special, and have
-     * missing clock pulses, allowing them to be found by the logic.
-     * 
-     * IAM record:
-     * flux:   XXXX-XXX-XXXX-X- = 0xf77a
-     * clock:  X X - X - X X X  = 0xd7
-     * data:    X X X X X X - - = 0xfc
-     * 
-     * IDAM record:
-     * flux:   XXXX-X-X-XXXXXX- = 0xf57e
-     * clock:  X X - - - X X X  = 0xc7
-     * data:    X X X X X X X - = 0xfe
-     * 
-     * DAM1 record:
-     * flux:   XXXX-X-X-XX-X-X- = 0xf56a
-     * clock:  X X - - - X X X  = 0xc7
-     * data:    X X X X X - - - = 0xf8
-     * 
-     * DAM2 record:
-     * flux:   XXXX-X-X-XX-XXXX = 0xf56f
-     * clock:  X X - - - X X X  = 0xc7
-     * data:    X X X X X - X X = 0xfb
-     * 
-     * TRS80DAM1 record:
-     * flux:   XXXX-X-X-XX-X-XX = 0xf56b
-     * clock:  X X - - - X X X  = 0xc7
-     * data:    X X X X X - - X = 0xf9
-     * 
-     * TRS80DAM2 record:
-     * flux:   XXXX-X-X-XX-XXX- = 0xf56c
-     * clock:  X X - - - X X X  = 0xc7
-     * data:    X X X X X - X - = 0xfa
-     */
-         
-    uint16_t masked = fifo & 0xffff;
-    switch (masked)
+    /* If this is the MFM prefix byte, the the decoder is going to expect three
+     * extra bytes on the front of the header. */
+    _currentHeaderLength = (matcher == &MFM_PATTERN) ? 3 : 0;
+
+    Fluxmap::Position here = tell();
+    if (_currentHeaderLength > 0)
+        readRawBits(_currentHeaderLength*16);
+    auto idbits = readRawBits(16);
+    uint8_t id = decodeFmMfm(idbits).slice(0, 1)[0];
+    seek(here);
+    
+    switch (id)
     {
-        case 0xf77a:
-        case 0xf57e:
-        case 0xf56a:
-        case 0xf56f:
-        case 0xf56b:
-        case 0xf56c:
-            return 16;
-        
-        default:
-            return 0;
+        case IBM_IDAM:
+            return RecordType::SECTOR_RECORD;
+
+        case IBM_DAM1:
+        case IBM_DAM2:
+        case IBM_TRS80DAM1:
+        case IBM_TRS80DAM2:
+            return RecordType::DATA_RECORD;
     }
+    return RecordType::UNKNOWN_RECORD;
 }
 
-int IbmMfmDecoder::recordMatcher(uint64_t fifo) const
+void IbmDecoder::decodeSectorRecord()
 {
-    /* 
-     * The IAM record, which is the first one on the disk (and is optional), uses
-     * a distorted 0xC2 0xC2 0xC2 marker to identify it. Unfortunately, if this is
-     * shifted out of phase, it becomes a legal encoding, so if we're looking at
-     * real data we can't honour this. It can easily be read by keeping state as
-     * to whether we're reading or seeking, but it's completely useless and so I
-     * can't be bothered.
-     * 
-     * 0xC2 is:
-     * data:    1  1  0  0  0  0  1 0
-     * mfm:     01 01 00 10 10 10 01 00 = 0x5254
-     * special: 01 01 00 10 00 10 01 00 = 0x5224
-     *                    ^^^^
-     * shifted: 10 10 01 00 01 00 10 0. = legal, and might happen in real data
-     * 
-     * Therefore, when we've read the marker, the input fifo will contain
-     * 0xXXXX522252225222.
-     * 
-     * All other records use 0xA1 as a marker:
-     * 
-     * 0xA1  is:
-     * data:    1  0  1  0  0  0  0  1
-     * mfm:     01 00 01 00 10 10 10 01 = 0x44A9
-     * special: 01 00 01 00 10 00 10 01 = 0x4489
-     *                       ^^^^^
-     * shifted: 10 00 10 01 00 01 00 1
-     * 
-     * When this is shifted out of phase, we get an illegal encoding (you
-     * can't do 10 00). So, if we ever see 0x448944894489 in the input
-     * fifo, we know we've landed at the beginning of a new record.
-     */
-         
-    uint64_t masked = fifo & 0xffffffffffffLL;
-    if (masked == 0x448944894489LL)
-        return 48;
-    return 0;
+    unsigned recordSize = _currentHeaderLength + IBM_IDAM_LEN;
+    auto bits = readRawBits(recordSize*16);
+    auto bytes = decodeFmMfm(bits).slice(0, recordSize);
+
+    ByteReader br(bytes);
+    br.seek(_currentHeaderLength);
+    br.read_8(); /* skip ID byte */
+    _sector->logicalTrack = br.read_8();
+    _sector->logicalSide = br.read_8();
+    _sector->logicalSector = br.read_8() - _sectorBase;
+    _currentSectorSize = 1 << (br.read_8() + 7);
+    uint16_t wantCrc = br.read_be16();
+    uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, _currentHeaderLength + 5));
+    if (wantCrc == gotCrc)
+        _sector->status = Sector::DATA_MISSING; /* correct but unintuitive */
+}
+
+void IbmDecoder::decodeDataRecord()
+{
+    unsigned recordLength = _currentHeaderLength + _currentSectorSize + 3;
+    auto bits = readRawBits(recordLength*16);
+    auto bytes = decodeFmMfm(bits).slice(0, recordLength);
+
+    ByteReader br(bytes);
+    br.seek(_currentHeaderLength);
+    br.read_8(); /* skip ID byte */
+
+    _sector->data = br.read(_currentSectorSize);
+    uint16_t wantCrc = br.read_be16();
+    uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, recordLength-2));
+    _sector->status = (wantCrc == gotCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/ibm/ibm.h

@@ -1,10 +1,11 @@
 #ifndef IBM_H
 #define IBM_H
 
-#include "decoders.h"
+#include "decoders/decoders.h"
 
 /* IBM format (i.e. ordinary PC floppies). */
 
+#define IBM_MFM_SYNC   0xA1   /* sync byte for MFM */
 #define IBM_IAM        0xFC   /* start-of-track record */
 #define IBM_IAM_LEN    1      /* plus prologue */
 #define IBM_IDAM       0xFE   /* sector header */
@@ -27,6 +28,24 @@ struct IbmIdam
     uint8_t crc[2];
 };
 
+class IbmDecoder : public AbstractDecoder
+{
+public:
+    IbmDecoder(unsigned sectorBase):
+        _sectorBase(sectorBase)
+    {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+
+private:
+    unsigned _sectorBase;
+    unsigned _currentSectorSize;
+    unsigned _currentHeaderLength;
+};
+
+#if 0
 class AbstractIbmDecoder : public AbstractSoftSectorDecoder
 {
 public:
@@ -35,7 +54,7 @@ public:
     {}
     virtual ~AbstractIbmDecoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
+    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack, unsigned physicalSide);
 
 protected:
     virtual int skipHeaderBytes() const = 0;
@@ -72,5 +91,6 @@ protected:
     int skipHeaderBytes() const
     { return 3; }
 };
+#endif
 
 #endif

lib/image.cc

@@ -41,8 +41,13 @@ void readSectorsFromFile(SectorSet& sectors, const Geometry& geometry,
 				Bytes data(geometry.sectorSize);
 				inputFile.read((char*) data.begin(), geometry.sectorSize);
 
-				sectors.get(track, head, sectorId).reset(
-					new Sector(Sector::OK, track, head, sectorId, data));
+				std::unique_ptr<Sector>& sector = sectors.get(track, head, sectorId);
+				sector.reset(new Sector);
+				sector->status = Sector::OK;
+				sector->logicalTrack = sector->physicalTrack = track;
+				sector->logicalSide = sector->physicalSide = head;
+				sector->logicalSector = sectorId;
+				sector->data = data;
 			}
 		}
 	}
@@ -64,7 +69,7 @@ void writeSectorsToFile(const SectorSet& sectors, const Geometry& geometry,
 			std::cout << fmt::format("{}.{:2} ", head, sectorId);
 			for (int track = 0; track < geometry.tracks; track++)
 			{
-				auto sector = sectors.get(track, head, sectorId);
+				Sector* sector = sectors.get(track, head, sectorId);
 				if (!sector)
 				{
 					std::cout << 'X';
@@ -136,8 +141,8 @@ void writeSectorsToFile(const SectorSet& sectors, const Geometry& geometry,
 				auto sector = sectors.get(track, head, sectorId);
 				if (sector)
 				{
-					outputFile.seekp(sector->track*trackSize + sector->side*headSize + sector->sector*geometry.sectorSize, std::ios::beg);
-					outputFile.write((const char*) &sector->data[0], sector->data.size());
+					outputFile.seekp(sector->logicalTrack*trackSize + sector->logicalSide*headSize + sector->logicalSector*geometry.sectorSize, std::ios::beg);
+					outputFile.write((const char*) sector->data.cbegin(), sector->data.size());
 				}
 			}
 		}

lib/macintosh/decoder.cc

@@ -1,15 +1,21 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
+#include "track.h"
 #include "macintosh.h"
 #include "bytes.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
+const FluxPattern SECTOR_RECORD_PATTERN(24, MAC_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, MAC_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
@@ -25,7 +31,7 @@ static int decode_data_gcr(uint8_t gcr)
 /* This is extremely inspired by the MESS implementation, written by Nathan Woods
  * and R. Belmont: https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap_dsk35.cpp
  */
-static Bytes decode_crazy_data(const Bytes& input, int& status)
+static Bytes decode_crazy_data(const Bytes& input, Sector::Status& status)
 {
     Bytes output;
     ByteWriter bw(output);
@@ -118,74 +124,58 @@ uint8_t decode_side(uint8_t side)
     return !!(side & 0x40);
 }
 
-SectorVector MacintoshDecoder::decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack)
+AbstractDecoder::RecordType MacintoshDecoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-    int nextSector;
-    int nextSide;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const std::vector<bool>& rawdata = rawrecord->data;
-        const auto& rawbytes = toBytes(rawdata);
-
-        if (rawbytes.size() < 8)
-            continue;
-
-        uint32_t signature = rawbytes.reader().read_be24();
-        switch (signature)
-        {
-            case MAC_SECTOR_RECORD:
-            {
-                unsigned track = decode_data_gcr(rawbytes[3]);
-                if (track != (physicalTrack & 0x3f))
-                    break;
-                nextSector = decode_data_gcr(rawbytes[4]);
-                nextSide = decode_data_gcr(rawbytes[5]);
-                uint8_t formatByte = decode_data_gcr(rawbytes[6]);
-                uint8_t wantedsum = decode_data_gcr(rawbytes[7]);
-
-                uint8_t gotsum = (track ^ nextSector ^ nextSide ^ formatByte) & 0x3f;
-                headerIsValid = (wantedsum == gotsum);
-                break;
-            }
-
-            case MAC_DATA_RECORD:
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-
-                Bytes inputbuffer(MAC_ENCODED_SECTOR_LENGTH + 4);
-                for (unsigned i=0; i<inputbuffer.size(); i++)
-                {
-                    auto p = rawbytes.begin() + 4 + i;
-                    if (p > rawbytes.end())
-                        break;
-                    
-                    inputbuffer[i] = decode_data_gcr(*p);
-                }
-                    
-                int status = Sector::BAD_CHECKSUM;
-                auto data = decode_crazy_data(inputbuffer, status);
-
-                auto sector = std::unique_ptr<Sector>(
-                    new Sector(status, physicalTrack, decode_side(nextSide), nextSector, data));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return DATA_RECORD;
+	return UNKNOWN_RECORD;
 }
 
-int MacintoshDecoder::recordMatcher(uint64_t fifo) const
+void MacintoshDecoder::decodeSectorRecord()
 {
-    uint32_t masked = fifo & 0xffffff;
-    if ((masked == MAC_SECTOR_RECORD) || (masked == MAC_DATA_RECORD))
-		return 24;
-    return 0;
+    /* Skip ID (as we know it's a MAC_SECTOR_RECORD). */
+    readRawBits(24);
+
+    /* Read header. */
+
+    auto header = toBytes(readRawBits(7*8)).slice(0, 7);
+                
+    uint8_t encodedTrack = decode_data_gcr(header[0]);
+    if (encodedTrack != (_track->physicalTrack & 0x3f))
+        return;
+                
+    uint8_t encodedSector = decode_data_gcr(header[1]);
+    uint8_t encodedSide = decode_data_gcr(header[2]);
+    uint8_t formatByte = decode_data_gcr(header[3]);
+    uint8_t wantedsum = decode_data_gcr(header[4]);
+
+    _sector->logicalTrack = _track->physicalTrack;
+    _sector->logicalSide = decode_side(encodedSide);
+    _sector->logicalSector = encodedSector;
+    uint8_t gotsum = (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;
+    if (wantedsum == gotsum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void MacintoshDecoder::decodeDataRecord()
+{
+    auto id = toBytes(readRawBits(24)).reader().read_be24();
+    if (id != MAC_DATA_RECORD)
+        return;
+
+    /* Read data. */
+
+    readRawBits(8); /* skip spare byte */
+    auto inputbuffer = toBytes(readRawBits(MAC_ENCODED_SECTOR_LENGTH*8))
+        .slice(0, MAC_ENCODED_SECTOR_LENGTH);
+
+    for (unsigned i=0; i<inputbuffer.size(); i++)
+        inputbuffer[i] = decode_data_gcr(inputbuffer[i]);
+        
+    _sector->status = Sector::BAD_CHECKSUM;
+    _sector->data = decode_crazy_data(inputbuffer, _sector->status);
 }

lib/macintosh/macintosh.h

@@ -1,22 +1,23 @@
 #ifndef MACINTOSH_H
 #define MACINTOSH_H
 
-#define MAC_SECTOR_RECORD   0xd5aa96
-#define MAC_DATA_RECORD     0xd5aaad
+#define MAC_SECTOR_RECORD   0xd5aa96 /* 1101 0101 1010 1010 1001 0110 */
+#define MAC_DATA_RECORD     0xd5aaad /* 1101 0101 1010 1010 1010 1101 */
 
 #define MAC_SECTOR_LENGTH   524 /* yes, really */
-#define MAC_ENCODED_SECTOR_LENGTH 700
+#define MAC_ENCODED_SECTOR_LENGTH 703
 
 class Sector;
 class Fluxmap;
 
-class MacintoshDecoder : public AbstractSoftSectorDecoder
+class MacintoshDecoder : public AbstractDecoder
 {
 public:
     virtual ~MacintoshDecoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
 #endif

lib/mx/decoder.cc

@@ -0,0 +1,75 @@
+#include "globals.h"
+#include "decoders/decoders.h"
+#include "mx/mx.h"
+#include "crc.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "sector.h"
+#include "record.h"
+#include "track.h"
+#include <string.h>
+
+const int SECTOR_SIZE = 256;
+
+/*
+ * MX disks are a bunch of sectors glued together with no gaps or sync markers,
+ * following a single beginning-of-track synchronisation and identification
+ * sequence.
+ */
+
+/* FM beginning of track marker:
+ * 1010 1010 1010 1010 1111 1111 1010 1111
+ *    a    a    a    a    f    f    a    f
+ */
+const FluxPattern ID_PATTERN(32, 0xaaaaffaf);
+
+void MxDecoder::beginTrack()
+{
+    _currentSector = -1;
+    _clock = 0;
+}
+
+AbstractDecoder::RecordType MxDecoder::advanceToNextRecord()
+{
+    if (_currentSector == -1)
+    {
+        /* First sector in the track: look for the sync marker. */
+        const FluxMatcher* matcher = nullptr;
+        _sector->clock = _clock = _fmr->seekToPattern(ID_PATTERN, matcher);
+        readRawBits(32); /* skip the ID mark */
+        readRawBits(32); /* skip the track number */
+    }
+    else if (_currentSector == 10)
+    {
+        /* That was the last sector on the disk. */
+        return UNKNOWN_RECORD;
+    }
+    else
+    {
+        /* Otherwise we assume the clock from the first sector is still valid.
+         * The decoder framwork will automatically stop when we hit the end of
+         * the track. */
+        _sector->clock = _clock;
+    }
+
+    _currentSector++;
+    return SECTOR_RECORD;
+}
+
+void MxDecoder::decodeSectorRecord()
+{
+    auto bits = readRawBits((SECTOR_SIZE+2)*16);
+    auto bytes = decodeFmMfm(bits).slice(0, SECTOR_SIZE+2).swab();
+
+    uint16_t gotChecksum = 0;
+    ByteReader br(bytes);
+    for (int i=0; i<(SECTOR_SIZE/2); i++)
+        gotChecksum += br.read_le16();
+    uint16_t wantChecksum = br.read_le16();
+
+    _sector->logicalTrack = _track->physicalTrack;
+    _sector->logicalSide = _track->physicalSide;
+    _sector->logicalSector = _currentSector;
+    _sector->data = bytes.slice(0, SECTOR_SIZE);
+    _sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

lib/mx/mx.h

@@ -0,0 +1,20 @@
+#ifndef MX_H
+#define MX_H
+
+#include "decoders/decoders.h"
+
+class MxDecoder : public AbstractDecoder
+{
+public:
+    virtual ~MxDecoder() {}
+
+    void beginTrack();
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+
+private:
+    nanoseconds_t _clock;
+    int _currentSector;
+};
+
+#endif

lib/reader.cc

@@ -1,20 +1,23 @@
 #include "globals.h"
 #include "flags.h"
 #include "usb.h"
-#include "fluxreader.h"
+#include "fluxsource/fluxsource.h"
 #include "reader.h"
 #include "fluxmap.h"
 #include "sql.h"
 #include "dataspec.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "sectorset.h"
 #include "record.h"
 #include "image.h"
 #include "bytes.h"
-#include "rawbits.h"
+#include "decoders/rawbits.h"
+#include "track.h"
 #include "fmt/format.h"
 
+FlagGroup readerFlags { &hardwareFluxSourceFlags, &fluxmapReaderFlags };
+
 static DataSpecFlag source(
     { "--source", "-s" },
     "source for data",
@@ -38,6 +41,10 @@ static IntFlag retries(
 	"How many times to retry each track in the event of a read failure.",
 	5);
 
+static SettableFlag highDensityFlag(
+	{ "--high-density", "--hd" },
+	"set the drive to high density mode");
+
 static sqlite3* outdb;
 
 void setReaderDefaultSource(const std::string& source)
@@ -47,43 +54,33 @@ void setReaderDefaultSource(const std::string& source)
 
 void setReaderRevolutions(int revolutions)
 {
-	setHardwareFluxReaderRevolutions(revolutions);
+	setHardwareFluxSourceRevolutions(revolutions);
 }
 
-std::unique_ptr<Fluxmap> Track::read()
+void Track::readFluxmap()
 {
-	std::cout << fmt::format("{0:>3}.{1}: ", track, side) << std::flush;
-	std::unique_ptr<Fluxmap> fluxmap = _fluxReader->readFlux(track, side);
+	std::cout << fmt::format("{0:>3}.{1}: ", physicalTrack, physicalSide) << std::flush;
+	fluxmap = fluxsource->readFlux(physicalTrack, physicalSide);
 	std::cout << fmt::format(
-		"{0} ms in {1} bytes ({2} kB/s)\n",
+		"{0} ms in {1} bytes\n",
             int(fluxmap->duration()/1e6),
-            fluxmap->bytes(),
-            (int)(1e6 * fluxmap->bytes() / fluxmap->duration()));
+            fluxmap->bytes());
 	if (outdb)
-		sqlWriteFlux(outdb, track, side, *fluxmap);
-	return fluxmap;
-}
-
-void Track::recalibrate()
-{
-	_fluxReader->recalibrate();
-}
-
-bool Track::retryable()
-{
-	return _fluxReader->retryable();
+		sqlWriteFlux(outdb, physicalTrack, physicalSide, *fluxmap);
 }
 
 std::vector<std::unique_ptr<Track>> readTracks()
 {
-    const DataSpec& dataSpec = source.value;
+    const DataSpec& dataSpec = source;
 
     std::cout << "Reading from: " << dataSpec << std::endl;
 
-	if (!destination.value.empty())
+	setHardwareFluxSourceDensity(highDensityFlag);
+
+	if (!destination.get().empty())
 	{
 		outdb = sqlOpen(destination, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
-		std::cout << "Writing a copy of the flux to " << destination.value << std::endl;
+		std::cout << "Writing a copy of the flux to " << destination.get() << std::endl;
 		sqlPrepareFlux(outdb);
 		sqlStmt(outdb, "BEGIN;");
         sqlWriteIntProperty(outdb, "version", FLUX_VERSION_CURRENT);
@@ -95,17 +92,20 @@ std::vector<std::unique_ptr<Track>> readTracks()
 		);
 	}
 
-	std::shared_ptr<FluxReader> fluxreader = FluxReader::create(dataSpec);
+	std::shared_ptr<FluxSource> fluxSource = FluxSource::create(dataSpec);
 
 	std::vector<std::unique_ptr<Track>> tracks;
     for (const auto& location : dataSpec.locations)
-		tracks.push_back(
-			std::unique_ptr<Track>(new Track(fluxreader, location.track, location.side)));
+	{
+		auto track = std::make_unique<Track>(location.track, location.side);
+		track->fluxsource = fluxSource;
+		tracks.push_back(std::move(track));
+	}
 
 	if (justRead)
 	{
 		for (auto& track : tracks)
-			track->read();
+			track->readFluxmap();
 		
 		std::cout << "--just-read specified, halting now" << std::endl;
 		exit(0);
@@ -114,97 +114,85 @@ std::vector<std::unique_ptr<Track>> readTracks()
 	return tracks;
 }
 
-static bool conflictable(int status)
+static bool conflictable(Sector::Status status)
 {
 	return (status == Sector::OK) || (status == Sector::CONFLICT);
 }
 
-static void replace_sector(std::unique_ptr<Sector>& replacing, std::unique_ptr<Sector>& replacement)
+static void replace_sector(std::unique_ptr<Sector>& replacing, Sector& replacement)
 {
-	if (replacing && conflictable(replacing->status) && conflictable(replacement->status))
+	if (replacing && conflictable(replacing->status) && conflictable(replacement.status))
 	{
-		if (replacement->data != replacing->data)
+		if (replacement.data != replacing->data)
 		{
 			std::cout << std::endl
-						<< "       multiple conflicting copies of sector " << replacing->sector
+						<< "       multiple conflicting copies of sector " << replacing->logicalSector
 						<< " seen; ";
 			replacing->status = Sector::CONFLICT;
 			return;
 		}
 	}
 	if (!replacing || (replacing->status != Sector::OK))
-		replacing = std::move(replacement);
+	{
+		if (!replacing)
+			replacing.reset(new Sector);
+		*replacing = replacement;
+	}
 }
 
 void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename)
 {
 	bool failures = false;
 	SectorSet allSectors;
-    for (const auto& track : readTracks())
+	auto tracks = readTracks();
+    for (const auto& track : tracks)
 	{
 		std::map<int, std::unique_ptr<Sector>> readSectors;
 		for (int retry = ::retries; retry >= 0; retry--)
 		{
-			std::unique_ptr<Fluxmap> fluxmap = track->read();
+			track->readFluxmap();
+			decoder.decodeToSectors(*track);
 
-			nanoseconds_t clockPeriod = decoder.guessClock(*fluxmap);
-			if (clockPeriod == 0)
+			std::cout << "       ";
+			if (!track->sectors.empty())
 			{
-				std::cout << "       no clock detected; giving up" << std::endl;
-				continue;
-			}
-			std::cout << fmt::format("       {:.2f} us clock; ", (double)clockPeriod/1000.0) << std::flush;
+				std::cout << fmt::format("{} records, {} sectors; ",
+					track->rawrecords.size(),
+					track->sectors.size());
+				if (track->sectors.size() > 0)
+					std::cout << fmt::format("{:.2f}us clock; ",
+						track->sectors.begin()->clock / 1000.0);
 
-			const auto& bitmap = fluxmap->decodeToBits(clockPeriod);
-			std::cout << fmt::format("{} bytes encoded; ", bitmap.size()/8) << std::flush;
-
-			auto rawrecords = decoder.extractRecords(bitmap);
-			std::cout << fmt::format("{} records", rawrecords.size()) << std::endl;
-
-			auto sectors = decoder.decodeToSectors(rawrecords, track->track);
-			std::cout << "       " << sectors.size() << " sectors; ";
-
-			for (auto& sector : sectors)
-			{
-                auto& replacing = readSectors[sector->sector];
-				replace_sector(replacing, sector);
-			}
-
-			bool hasBadSectors = false;
-			for (const auto& i : readSectors)
-			{
-                const auto& sector = i.second;
-				if (sector->status != Sector::OK)
+				for (auto& sector : track->sectors)
 				{
-					std::cout << std::endl
-							  << "       Failed to read sector " << sector->sector
-                              << " (" << Sector::statusToString((Sector::Status)sector->status) << "); ";
-					hasBadSectors = true;
+					auto& replacing = readSectors[sector.logicalSector];
+					replace_sector(replacing, sector);
 				}
-			}
 
-			if (hasBadSectors)
-				failures = false;
-
-			if (dumpRecords && (!hasBadSectors || (retry == 0) || !track->retryable()))
-			{
-				std::cout << "\nRaw (undecoded) records follow:\n\n";
-				for (auto& record : rawrecords)
+				bool hasBadSectors = false;
+				for (const auto& i : readSectors)
 				{
-					std::cout << fmt::format("I+{:.3f}ms", (double)(record->position*clockPeriod)/1e6)
-					          << std::endl;
-					hexdump(std::cout, toBytes(record->data));
-					std::cout << std::endl;
+					const auto& sector = i.second;
+					if (sector->status != Sector::OK)
+					{
+						std::cout << std::endl
+								<< "       Failed to read sector " << sector->logicalSector
+								<< " (" << Sector::statusToString((Sector::Status)sector->status) << "); ";
+						hasBadSectors = true;
+					}
 				}
+
+				if (hasBadSectors)
+					failures = false;
+
+				std::cout << std::endl
+						<< "       ";
+
+				if (!hasBadSectors)
+					break;
 			}
 
-			std::cout << std::endl
-                      << "       ";
-
-			if (!hasBadSectors)
-				break;
-
-			if (!track->retryable())
+			if (!track->fluxsource->retryable())
 				break;
             if (retry == 0)
                 std::cout << "giving up" << std::endl
@@ -212,10 +200,22 @@ void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename
             else
             {
 				std::cout << retry << " retries remaining" << std::endl;
-                track->recalibrate();
+                track->fluxsource->recalibrate();
             }
 		}
 
+		if (dumpRecords)
+		{
+			std::cout << "\nRaw (undecoded) records follow:\n\n";
+			for (auto& record : track->rawrecords)
+			{
+				std::cout << fmt::format("I+{:.2f}us", record.position.ns() / 1000.0)
+						<< std::endl;
+				hexdump(std::cout, record.data);
+				std::cout << std::endl;
+			}
+		}
+
         int size = 0;
 		bool printedTrack = false;
         for (auto& i : readSectors)
@@ -225,14 +225,14 @@ void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename
 			{
 				if (!printedTrack)
 				{
-					std::cout << fmt::format("logical track {}.{}; ", sector->track, sector->side);
+					std::cout << fmt::format("logical track {}.{}; ", sector->logicalTrack, sector->logicalSide);
 					printedTrack = true;
 				}
 
 				size += sector->data.size();
 
-				auto& replacing = allSectors.get(sector->track, sector->side, sector->sector);
-				replace_sector(replacing, sector);
+				std::unique_ptr<Sector>& replacing = allSectors.get(sector->logicalTrack, sector->logicalSide, sector->logicalSector);
+				replace_sector(replacing, *sector);
 			}
         }
         std::cout << size << " bytes decoded." << std::endl;

lib/reader.h

@@ -1,34 +1,18 @@
 #ifndef READER_H
 #define READER_H
 
+#include "flags.h"
+
 class Fluxmap;
-class FluxReader;
+class FluxSource;
 class AbstractDecoder;
+class Track;
+
+extern FlagGroup readerFlags;
 
 extern void setReaderDefaultSource(const std::string& source);
 extern void setReaderRevolutions(int revolutions);
 
-class Track
-{
-public:
-    Track(std::shared_ptr<FluxReader>& fluxReader, unsigned track, unsigned side):
-        track(track),
-        side(side),
-        _fluxReader(fluxReader)
-    {}
-
-public:
-    std::unique_ptr<Fluxmap> read();
-    void recalibrate();
-    bool retryable();
-
-    unsigned track;
-    unsigned side;
-
-private:
-    std::shared_ptr<FluxReader> _fluxReader;
-};
-
 extern std::vector<std::unique_ptr<Track>> readTracks();
 
 extern void readDiskCommand(AbstractDecoder& decoder, const std::string& outputFilename);

lib/record.h

@@ -1,21 +1,19 @@
 #ifndef RECORD_H
 #define RECORD_H
 
+#include "fluxmap.h"
+
 class RawRecord
 {
 public:
-	RawRecord(
-			size_t position,
-			const std::vector<bool>::const_iterator start,
-			const std::vector<bool>::const_iterator end):
-		position(position), data(start, end)
-	{}
+	RawRecord() {}
 
-	size_t position; // in bits
-	std::vector<bool> data;
+	Fluxmap::Position position;
+	nanoseconds_t clock = 0;
+    int physicalTrack = 0;
+    int physicalSide = 0;
+	Bytes data;
 };
 
-typedef std::vector<std::unique_ptr<RawRecord>> RawRecordVector;
-
 #endif
 

lib/sector.cc

@@ -9,6 +9,8 @@ const std::string Sector::statusToString(Status status)
         case Status::OK:           return "OK";
         case Status::BAD_CHECKSUM: return "bad checksum";
         case Status::MISSING:      return "sector not found";
+        case Status::DATA_MISSING: return "present but no data found";
+        case Status::CONFLICT:     return "conflicting data";
         default:                   return fmt::format("unknown error {}", status);
     }
 }

lib/sector.h

@@ -2,6 +2,7 @@
 #define SECTOR_H
 
 #include "bytes.h"
+#include "fluxmap.h"
 
 /* 
  * Note that sectors here used zero-based numbering throughout (to make the
@@ -16,27 +17,23 @@ public:
 		OK,
 		BAD_CHECKSUM,
         MISSING,
-        CONFLICT
+        DATA_MISSING,
+        CONFLICT,
+        INTERNAL_ERROR
 	};
 
     static const std::string statusToString(Status status);
 
-    Sector(int status, int track, int side, int sector, const Bytes& data):
-		status(status),
-        track(track),
-        side(side),
-        sector(sector),
-        data(data)
-    {}
-
-	int status;
-    const int track;
-    const int side;
-    const int sector;
-    const Bytes data;
+	Status status = Status::INTERNAL_ERROR;
+    Fluxmap::Position position;
+    nanoseconds_t clock = 0;
+    int physicalTrack = 0;
+    int physicalSide = 0;
+    int logicalTrack = 0;
+    int logicalSide = 0;
+    int logicalSector = 0;
+    Bytes data;
 };
 
-typedef std::vector<std::unique_ptr<Sector>> SectorVector;
-
 #endif
 

lib/sectorset.cc

@@ -5,13 +5,13 @@
 
 std::unique_ptr<Sector>& SectorSet::get(int track, int head, int sector)
 {
-        key_t key(track, head, sector);
+	key_t key(track, head, sector);
 	return _data[key];
 }
 
 Sector* SectorSet::get(int track, int head, int sector) const
 {
-        key_t key(track, head, sector);
+    key_t key(track, head, sector);
 	auto i = _data.find(key);
 	if (i == _data.end())
 		return NULL;
@@ -28,9 +28,9 @@ void SectorSet::calculateSize(int& numTracks, int& numHeads, int& numSectors,
 		auto& sector = i.second;
 		if (sector)
 		{
-			numTracks = std::max(numTracks, sector->track+1);
-			numHeads = std::max(numHeads, sector->side+1);
-			numSectors = std::max(numSectors, sector->sector+1);
+			numTracks = std::max(numTracks, sector->logicalTrack+1);
+			numHeads = std::max(numHeads, sector->logicalSide+1);
+			numSectors = std::max(numSectors, sector->logicalSector+1);
 			sectorSize = std::max(sectorSize, (int)sector->data.size());
 		}
 	}

lib/track.h

@@ -0,0 +1,30 @@
+#ifndef TRACK_H
+#define TRACK_H
+
+class Fluxmap;
+class FluxSource;
+class AbstractDecoder;
+
+class Track
+{
+public:
+    Track(unsigned track, unsigned side):
+        physicalTrack(track),
+        physicalSide(side)
+    {}
+
+    void readFluxmap();
+
+public:
+    unsigned physicalTrack;
+    unsigned physicalSide;
+    std::shared_ptr<FluxSource> fluxsource;
+    std::unique_ptr<Fluxmap> fluxmap;
+
+    std::vector<RawRecord> rawrecords;
+    std::vector<Sector> sectors;
+};
+
+typedef std::vector<std::unique_ptr<Track>> TrackVector;
+
+#endif

lib/usb.cc

@@ -3,7 +3,9 @@
 #include "protocol.h"
 #include "fluxmap.h"
 #include "bytes.h"
+#include "common/crunch.h"
 #include <libusb.h>
+#include "fmt/format.h"
 
 #define TIMEOUT 5000
 
@@ -51,6 +53,8 @@ static void usb_init()
 
 static int usb_cmd_send(void* ptr, int len)
 {
+    //std::cerr << "send:\n";
+    //hexdump(std::cerr, Bytes((const uint8_t*)ptr, len));
     int i = libusb_interrupt_transfer(device, FLUXENGINE_CMD_OUT_EP,
         (uint8_t*) ptr, len, &len, TIMEOUT);
     if (i < 0)
@@ -64,13 +68,15 @@ void usb_cmd_recv(void* ptr, int len)
        (uint8_t*)  ptr, len, &len, TIMEOUT);
     if (i < 0)
         Error() << "failed to receive command reply: " << usberror(i);
+    //std::cerr << "recv:\n";
+    //hexdump(std::cerr, Bytes((const uint8_t*)ptr, len));
 }
 
 static void bad_reply(void)
 {
     struct error_frame* f = (struct error_frame*) buffer;
     if (f->f.type != F_FRAME_ERROR)
-        Error() << "bad USB reply " << f->f.type;
+        Error() << fmt::format("bad USB reply 0x{:2x}", f->f.type);
     switch (f->error)
     {
         case F_ERROR_BAD_COMMAND:
@@ -80,18 +86,26 @@ static void bad_reply(void)
             Error() << "USB underrun (not enough bandwidth)";
             
         default:
-            Error() << "unknown device error " << f->error;
+            Error() << fmt::format("unknown device error {}", f->error);
     }
 }
 
 template <typename T>
 static T* await_reply(int desired)
 {
-    usb_cmd_recv(buffer, sizeof(buffer));
-    struct any_frame* r = (struct any_frame*) buffer;
-    if (r->f.type != desired)
-        bad_reply();
-    return (T*) r;
+    for (;;)
+    {
+        usb_cmd_recv(buffer, sizeof(buffer));
+        struct any_frame* r = (struct any_frame*) buffer;
+        if (r->f.type == F_FRAME_DEBUG)
+        {
+            std::cout << "dev: " << ((struct debug_frame*)r)->payload << std::endl;
+            continue;
+        }
+        if (r->f.type != desired)
+            bad_reply();
+        return (T*) r;
+    }
 }
 
 int usbGetVersion(void)
@@ -188,7 +202,7 @@ void usbTestBulkTransport()
     await_reply<struct any_frame>(F_FRAME_BULK_TEST_REPLY);
 }
 
-std::unique_ptr<Fluxmap> usbRead(int side, int revolutions)
+Bytes usbRead(int side, int revolutions)
 {
     struct read_frame f = {
         .f = { .type = F_FRAME_READ_CMD, .size = sizeof(f) },
@@ -203,20 +217,14 @@ std::unique_ptr<Fluxmap> usbRead(int side, int revolutions)
     int len = large_bulk_transfer(FLUXENGINE_DATA_IN_EP, buffer);
     buffer.resize(len);
 
-    fluxmap->appendBytes(buffer);
-
     await_reply<struct any_frame>(F_FRAME_READ_REPLY);
-    return fluxmap;
+    return buffer;
 }
 
-void usbWrite(int side, const Fluxmap& fluxmap)
+void usbWrite(int side, const Bytes& bytes)
 {
-    unsigned safelen = fluxmap.bytes() & ~(FRAME_SIZE-1);
-
-    /* Convert from intervals to absolute timestamps. */
-
-	Bytes buffer(fluxmap.rawBytes());
-    buffer.resize(safelen);
+    unsigned safelen = bytes.size() & ~(FRAME_SIZE-1);
+    Bytes safeBytes = bytes.slice(0, safelen);
 
     struct write_frame f = {
         .f = { .type = F_FRAME_WRITE_CMD, .size = sizeof(f) },
@@ -229,7 +237,7 @@ void usbWrite(int side, const Fluxmap& fluxmap)
 
     usb_cmd_send(&f, f.f.size);
 
-    large_bulk_transfer(FLUXENGINE_DATA_OUT_EP, buffer);
+    large_bulk_transfer(FLUXENGINE_DATA_OUT_EP, safeBytes);
     
     await_reply<struct any_frame>(F_FRAME_WRITE_REPLY);
 }
@@ -245,13 +253,13 @@ void usbErase(int side)
     await_reply<struct any_frame>(F_FRAME_ERASE_REPLY);
 }
 
-void usbSetDrive(int drive)
+void usbSetDrive(int drive, bool high_density)
 {
     usb_init();
 
     struct set_drive_frame f = {
         { .type = F_FRAME_SET_DRIVE_CMD, .size = sizeof(f) },
-        .drive = (uint8_t) drive
+        .drive_flags = (uint8_t)((drive ? DRIVE_1 : DRIVE_0) | (high_density ? DRIVE_HD : DRIVE_DD)),
     };
     usb_cmd_send(&f, f.f.size);
     await_reply<struct any_frame>(F_FRAME_SET_DRIVE_REPLY);

lib/usb.h

@@ -2,15 +2,16 @@
 #define USB_H
 
 class Fluxmap;
+class Bytes;
 
 extern int usbGetVersion();
 extern void usbRecalibrate();
 extern void usbSeek(int track);
 extern nanoseconds_t usbGetRotationalPeriod();
 extern void usbTestBulkTransport();
-extern std::unique_ptr<Fluxmap> usbRead(int side, int revolutions);
-extern void usbWrite(int side, const Fluxmap& fluxmap);
+extern Bytes usbRead(int side, int revolutions);
+extern void usbWrite(int side, const Bytes& bytes);
 extern void usbErase(int side);
-extern void usbSetDrive(int drive);
+extern void usbSetDrive(int drive, bool high_density);
 
 #endif

lib/victor9k/decoder.cc

@@ -1,16 +1,22 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "victor9k.h"
 #include "crc.h"
 #include "bytes.h"
+#include "track.h"
 #include "fmt/format.h"
 #include <string.h>
 #include <algorithm>
 
+const FluxPattern SECTOR_RECORD_PATTERN(32, VICTOR9K_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(32, VICTOR9K_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
 static int decode_data_gcr(uint8_t gcr)
 {
     switch (gcr)
@@ -48,74 +54,58 @@ static Bytes decode(const std::vector<bool>& bits)
     return output;
 }
 
-SectorVector Victor9kDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
+AbstractDecoder::RecordType Victor9kDecoder::advanceToNextRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
-    unsigned nextSector;
-    unsigned nextTrack;
-    unsigned nextSide;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decode(rawdata);
-
-        if (bytes.size() == 0)
-            continue;
-
-        switch (bytes[0])
-        {
-            case 7: /* sector record */
-            {
-                headerIsValid = false;
-                if (bytes.size() < 6)
-                    break;
-
-                uint8_t rawTrack = bytes[1];
-                nextSector = bytes[2];
-                uint8_t gotChecksum = bytes[3];
-
-                nextTrack = rawTrack & 0x7f;
-                nextSide = rawTrack >> 7;
-                uint8_t wantChecksum = bytes[1] + bytes[2];
-                if (wantChecksum != gotChecksum)
-                    break;
-                if ((nextSector > 20) || (nextTrack > 85) || (nextSide > 1))
-                    break;
-
-                headerIsValid = true;
-                break;
-            }
-            
-            case 8: /* data record */
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-                if (bytes.size() < VICTOR9K_SECTOR_LENGTH+3)
-                    break;
-
-                Bytes payload = bytes.slice(1, VICTOR9K_SECTOR_LENGTH);
-                uint16_t gotChecksum = sumBytes(payload);
-                uint16_t wantChecksum = bytes.reader().seek(VICTOR9K_SECTOR_LENGTH+1).read_le16();
-                int status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return DATA_RECORD;
+	return UNKNOWN_RECORD;
 }
 
-int Victor9kDecoder::recordMatcher(uint64_t fifo) const
+void Victor9kDecoder::decodeSectorRecord()
 {
-    uint32_t masked = fifo & 0xfffff;
-    if ((masked == VICTOR9K_SECTOR_RECORD) || (masked == VICTOR9K_DATA_RECORD))
-		return 9;
-    return 0;
+    /* Skip the sync marker bit. */
+    readRawBits(23);
+
+    /* Read header. */
+
+    auto bytes = decode(readRawBits(4*10)).slice(0, 4);
+
+    uint8_t rawTrack = bytes[1];
+    _sector->logicalSector = bytes[2];
+    uint8_t gotChecksum = bytes[3];
+
+    _sector->logicalTrack = rawTrack & 0x7f;
+    _sector->logicalSide = rawTrack >> 7;
+    uint8_t wantChecksum = bytes[1] + bytes[2];
+    if ((_sector->logicalSector > 20) || (_sector->logicalTrack > 85) || (_sector->logicalSide > 1))
+        return;
+                
+    if (wantChecksum == gotChecksum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Victor9kDecoder::decodeDataRecord()
+{
+    /* Skip the sync marker bit. */
+    readRawBits(23);
+
+    /* Read data. */
+
+    auto bytes = decode(readRawBits((VICTOR9K_SECTOR_LENGTH+5)*10))
+        .slice(0, VICTOR9K_SECTOR_LENGTH+5);
+    ByteReader br(bytes);
+
+    /* Check that this is actually a data record. */
+    
+    if (br.read_8() != 8)
+        return;
+
+    _sector->data = br.read(VICTOR9K_SECTOR_LENGTH);
+    uint16_t gotChecksum = sumBytes(_sector->data);
+    uint16_t wantChecksum = br.read_le16();
+    _sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/victor9k/victor9k.h

@@ -1,22 +1,22 @@
 #ifndef VICTOR9K_H
 #define VICTOR9K_H
 
-#define VICTOR9K_SECTOR_RECORD 0xffeab
-#define VICTOR9K_DATA_RECORD   0xffea4
+#define VICTOR9K_SECTOR_RECORD 0xfffffeab
+#define VICTOR9K_DATA_RECORD   0xfffffea4
 
 #define VICTOR9K_SECTOR_LENGTH 512
 
 class Sector;
 class Fluxmap;
 
-class Victor9kDecoder : public AbstractSoftSectorDecoder
+class Victor9kDecoder : public AbstractDecoder
 {
 public:
     virtual ~Victor9kDecoder() {}
 
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
 #endif

lib/writer.cc

@@ -6,13 +6,21 @@
 #include "protocol.h"
 #include "usb.h"
 #include "dataspec.h"
+#include "fluxsource/fluxsource.h"
+#include "fluxsink/fluxsink.h"
 #include "fmt/format.h"
 
+FlagGroup writerFlags { &hardwareFluxSourceFlags };
+
 static DataSpecFlag dest(
     { "--dest", "-d" },
     "destination for data",
     ":d=0:t=0-79:s=0-1");
 
+static SettableFlag highDensityFlag(
+	{ "--high-density", "-H" },
+	"set the drive to high density mode");
+
 static sqlite3* outdb;
 
 void setWriterDefaultDest(const std::string& dest)
@@ -23,10 +31,13 @@ void setWriterDefaultDest(const std::string& dest)
 void writeTracks(
 	const std::function<std::unique_ptr<Fluxmap>(int track, int side)> producer)
 {
-    const auto& spec = dest.value;
+    const DataSpec& spec = dest;
 
     std::cout << "Writing to: " << spec << std::endl;
 
+	setHardwareFluxSourceDensity(highDensityFlag);
+	setHardwareFluxSinkDensity(highDensityFlag);
+
 	if (!spec.filename.empty())
 	{
 		outdb = sqlOpen(spec.filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
@@ -61,8 +72,9 @@ void writeTracks(
                 sqlWriteFlux(outdb, location.track, location.side, *fluxmap);
             else
             {
+                Bytes crunched = fluxmap->rawBytes().crunch();
                 usbSeek(location.track);
-                usbWrite(location.side, *fluxmap);
+                usbWrite(location.side, crunched);
             }
             std::cout << fmt::format(
                 "{0} ms in {1} bytes", int(fluxmap->duration()/1e6), fluxmap->bytes()) << std::endl;

lib/writer.h

@@ -1,6 +1,10 @@
 #ifndef WRITER_H
 #define WRITER_H
 
+#include "flags.h"
+
+extern FlagGroup writerFlags;
+
 class Fluxmap;
 
 extern void setWriterDefaultDest(const std::string& dest);

lib/zilogmcz/decoder.cc

@@ -1,8 +1,9 @@
 #include "globals.h"
 #include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
 #include "protocol.h"
 #include "record.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 #include "sector.h"
 #include "zilogmcz.h"
 #include "bytes.h"
@@ -11,49 +12,37 @@
 #include <string.h>
 #include <algorithm>
 
-static std::vector<bool>::const_iterator find_start_of_data(const std::vector<bool>& rawbits)
+static const FluxPattern SECTOR_START_PATTERN(16, 0xaaab);
+
+AbstractDecoder::RecordType ZilogMczDecoder::advanceToNextRecord()
 {
-    uint8_t fifo = 0;
-    auto ii = rawbits.begin();
-
-    while (ii != rawbits.end())
-    {
-        fifo = (fifo << 1) | *ii++;
-        if (fifo == 0xab)
-            return ii-2;
-    }
-
-    return ii;
+	const FluxMatcher* matcher = nullptr;
+    _fmr->seekToIndexMark();
+	_sector->clock = _fmr->seekToPattern(SECTOR_START_PATTERN, matcher);
+	if (matcher == &SECTOR_START_PATTERN)
+		return SECTOR_RECORD;
+	return UNKNOWN_RECORD;
 }
 
-SectorVector ZilogMczDecoder::decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack)
+void ZilogMczDecoder::decodeSectorRecord()
 {
-    std::vector<std::unique_ptr<Sector>> sectors;
+    readRawBits(14);
 
-    for (auto& rawrecord : rawRecords)
-    {
-        auto start = find_start_of_data(rawrecord->data);
-        auto rawbytes = decodeFmMfm(start, rawrecord->data.cend());
-        if (rawbytes.size() < 134)
-            continue;
+    auto rawbits = readRawBits(140*16);
+    auto bytes = decodeFmMfm(rawbits).slice(0, 140);
+    ByteReader br(bytes);
 
-        uint8_t sectorid = rawbytes[0] & 0x1f;
-        uint8_t track = rawbytes[1] & 0x7f;
-        if (sectorid > 31)
-            continue;
-        if (track > 80)
-            continue;
+    _sector->logicalSector = br.read_8() & 0x1f;
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = br.read_8() & 0x7f;
+    if (_sector->logicalSector > 31)
+        return;
+    if (_sector->logicalTrack > 80)
+        return;
 
-        Bytes payload = rawbytes.slice(2, 132);
-        uint16_t wantChecksum = rawbytes.reader().seek(134).read_be16();
-        uint16_t gotChecksum = crc16(MODBUS_POLY, 0x0000, rawbytes.slice(0, 134));
+    _sector->data = br.read(132);
+    uint16_t wantChecksum = br.read_be16();
+    uint16_t gotChecksum = crc16(MODBUS_POLY, 0x0000, bytes.slice(0, 134));
 
-        int status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-        auto sector = std::unique_ptr<Sector>(
-            new Sector(status, track, 0, sectorid, payload));
-        sectors.push_back(std::move(sector));
-	}
-
-	return sectors;
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
 }

lib/zilogmcz/zilogmcz.h

@@ -4,12 +4,13 @@
 class Sector;
 class Fluxmap;
 
-class ZilogMczDecoder : public AbstractHardSectorDecoder
+class ZilogMczDecoder : public AbstractDecoder
 {
 public:
     virtual ~ZilogMczDecoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
 };
 
 #endif

meson.build

@@ -1,233 +0,0 @@
-project('fluxclient', 'cpp')
-add_global_arguments('--std=c++14', language: 'cpp')
-
-libusb = dependency('libusb-1.0')
-sqlite = dependency('sqlite3')
-zlib = dependency('zlib')
-
-fmtlib = declare_dependency(
-    link_with:
-        shared_library('fmtlib',
-            [
-                'dep/fmt/format.cc',
-                'dep/fmt/posix.cc'
-            ]
-        ),
-    include_directories:
-        include_directories('dep/fmt')
-)
-
-felib = declare_dependency(
-    link_with:
-        shared_library('felib',
-            [
-                'lib/crc.cc',
-                'lib/dataspec.cc',
-                'lib/hexdump.cc',
-                'lib/sectorset.cc',
-                'lib/flags.cc',
-                'lib/fluxmap.cc',
-                'lib/globals.cc',
-                'lib/image.cc',
-                'lib/sector.cc',
-                'lib/usb.cc',
-                'lib/bytes.cc',
-            ],
-            dependencies: [fmtlib, libusb, zlib]
-        ),
-    include_directories:
-        include_directories('lib')
-)
-
-sqllib = declare_dependency(
-    link_with:
-        shared_library('sqllib',
-            ['lib/sql.cc'],
-            dependencies: [fmtlib, felib, sqlite, zlib]
-        )
-)
-
-fluxreaderlib = declare_dependency(
-    link_with:
-        shared_library('fluxreaderlib',
-            [
-                'lib/fluxreader/fluxreader.cc',
-                'lib/fluxreader/sqlitefluxreader.cc',
-                'lib/fluxreader/hardwarefluxreader.cc',
-                'lib/fluxreader/streamfluxreader.cc',
-                'lib/fluxreader/kryoflux.cc',
-            ],
-            dependencies: [fmtlib, felib, sqllib, sqlite]
-        ),
-    include_directories:
-        include_directories('lib/fluxreader')
-)
-
-decoderlib = declare_dependency(
-    link_with:
-        shared_library('decoderlib',
-            [
-                'lib/decoders/decoders.cc',
-                'lib/decoders/fmmfm.cc',
-            ],
-            dependencies: [fmtlib, felib]
-        ),
-    include_directories:
-        include_directories('lib/decoders')
-)
-
-readerlib = declare_dependency(
-    link_with:
-        shared_library('readerlib',
-            ['lib/reader.cc'],
-            dependencies: [fmtlib, felib, sqllib, decoderlib, fluxreaderlib, sqlite])
-)
-                        
-writerlib = declare_dependency(
-    link_with:
-        shared_library('writerlib',
-            ['lib/writer.cc'],
-            dependencies: [fmtlib, felib, sqllib, sqlite]
-        )
-)
-
-encoderlib = declare_dependency(
-    link_with:
-        shared_library('encoderlib',
-            ['lib/encoder.cc'],
-            dependencies: [fmtlib, felib]
-        )
-)
-
-aeslanierdecoderlib = declare_dependency(
-    link_with:
-        shared_library('aeslanierdecoderlib',
-        [ 'lib/aeslanier/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/aeslanier')
-)
-
-amigadecoderlib = declare_dependency(
-    link_with:
-        shared_library('amigadecoderlib',
-        [ 'lib/amiga/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/amiga')
-)
-
-apple2decoderlib = declare_dependency(
-    link_with:
-        shared_library('apple2decoderlib',
-        [ 'lib/apple2/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/apple2')
-)
-
-brotherdecoderlib = declare_dependency(
-    link_with:
-        shared_library('brotherdecoderlib',
-        [ 'lib/brother/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib, sqlite]),
-    include_directories:
-        include_directories('lib/brother')
-)
-
-brotherencoderlib = declare_dependency(
-    link_with:
-        shared_library('brotherencoderlib',
-        [ 'lib/brother/encoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib, encoderlib]),
-    include_directories:
-        include_directories('lib/brother')
-)
-
-c64decoderlib = declare_dependency(
-    link_with:
-        shared_library('c64decoderlib',
-        [ 'lib/c64/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/c64')
-)
-
-f85decoderlib = declare_dependency(
-    link_with:
-        shared_library('f85decoderlib',
-        [ 'lib/f85/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/f85')
-)
-
-ibmdecoderlib = declare_dependency(
-    link_with:
-        shared_library('ibmdecoderlib',
-        [ 'lib/ibm/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/ibm')
-)
-
-macdecoderlib = declare_dependency(
-    link_with:
-        shared_library('macdecoderlib',
-        [ 'lib/macintosh/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/macintosh')
-)
-
-zilogmczdecoderlib = declare_dependency(
-    link_with:
-        shared_library('zilogmczdecoderlib',
-        [ 'lib/zilogmcz/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/zilogmcz')
-)
-
-victor9kdecoderlib = declare_dependency(
-    link_with:
-        shared_library('victor9kdecoderlib',
-        [ 'lib/victor9k/decoder.cc', ],
-        dependencies: [fmtlib, felib, decoderlib]),
-    include_directories:
-        include_directories('lib/victor9k')
-)
-
-executable('fe-erase',             ['src/fe-erase.cc'],             dependencies: [felib, writerlib])
-executable('fe-inspect',           ['src/fe-inspect.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib])
-executable('fe-readadfs',          ['src/fe-readadfs.cc'],          dependencies: [fmtlib, felib, decoderlib, readerlib, ibmdecoderlib])
-executable('fe-readaeslanier',     ['src/fe-readaeslanier.cc'],     dependencies: [fmtlib, felib, decoderlib, readerlib, aeslanierdecoderlib])
-executable('fe-readamiga',         ['src/fe-readamiga.cc'],         dependencies: [fmtlib, felib, decoderlib, readerlib, amigadecoderlib])
-executable('fe-readampro',         ['src/fe-readampro.cc'],         dependencies: [fmtlib, felib, decoderlib, readerlib, ibmdecoderlib])
-executable('fe-readapple2',        ['src/fe-readapple2.cc'],        dependencies: [fmtlib, felib, decoderlib, readerlib, apple2decoderlib])
-executable('fe-readbrother',       ['src/fe-readbrother.cc'],       dependencies: [fmtlib, felib, decoderlib, readerlib, brotherdecoderlib])
-executable('fe-readc64',           ['src/fe-readc64.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib, c64decoderlib])
-executable('fe-readdfs',           ['src/fe-readdfs.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib, ibmdecoderlib])
-executable('fe-readf85',           ['src/fe-readf85.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib, f85decoderlib])
-executable('fe-readibm',           ['src/fe-readibm.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib, ibmdecoderlib])
-executable('fe-readmac',           ['src/fe-readmac.cc'],           dependencies: [fmtlib, felib, decoderlib, readerlib, macdecoderlib])
-executable('fe-readzilogmcz',      ['src/fe-readzilogmcz.cc'],      dependencies: [fmtlib, felib, decoderlib, readerlib, zilogmczdecoderlib])
-executable('fe-readvictor9k',      ['src/fe-readvictor9k.cc'],      dependencies: [fmtlib, felib, decoderlib, readerlib, victor9kdecoderlib])
-executable('fe-rpm',               ['src/fe-rpm.cc'],               dependencies: [fmtlib, felib])
-executable('fe-seek',              ['src/fe-seek.cc'],              dependencies: [fmtlib, felib])
-executable('fe-testbulktransport', ['src/fe-testbulktransport.cc'], dependencies: [fmtlib, felib])
-executable('fe-upgradefluxfile',   ['src/fe-upgradefluxfile.cc'],   dependencies: [fmtlib, felib, sqllib, sqlite])
-executable('fe-writebrother',      ['src/fe-writebrother.cc'],      dependencies: [fmtlib, felib, writerlib, decoderlib, encoderlib, brotherencoderlib])
-executable('fe-writeflux',         ['src/fe-writeflux.cc'],         dependencies: [fmtlib, felib, readerlib, writerlib])
-executable('fe-writetestpattern',  ['src/fe-writetestpattern.cc'],  dependencies: [fmtlib, felib, writerlib])
-
-executable('brother120tool',       ['tools/brother120tool.cc'],     dependencies: [fmtlib, felib])
-executable('cwftoflux',            ['tools/cwftoflux.cc'],          dependencies: [fmtlib, felib, sqllib, sqlite])
-
-test('DataSpec',        executable('dataspec-test', ['tests/dataspec.cc'],             dependencies: [felib]))
-test('Flags',           executable('flags-test', ['tests/flags.cc'],                   dependencies: [felib]))
-test('FmMfm',           executable('fmmfm-test', ['tests/fmmfm.cc'],                   dependencies: [felib, decoderlib]))
-test('BitAccumulator',  executable('bitaccumulator-test', ['tests/bitaccumulator.cc'], dependencies: [felib]))
-test('Kryoflux',        executable('kryoflux-test', ['tests/kryoflux.cc'],             dependencies: [felib, decoderlib, fluxreaderlib]))
-test('Compression',     executable('compression-test', ['tests/compression.cc'],       dependencies: [felib, decoderlib]))
-test('Bytes',           executable('bytes-test', ['tests/bytes.cc'],                   dependencies: [felib]))

mkninja.sh

@@ -0,0 +1,194 @@
+#!/bin/sh
+set -e
+
+cat <<EOF
+rule cxx
+    command = $CXX $CFLAGS \$flags -I. -c -o \$out \$in -MMD -MF \$out.d
+    description = CXX \$in
+    depfile = \$out.d
+    deps = gcc
+    
+rule library
+    command = $AR \$out \$in
+    description = AR \$in
+
+rule link
+    command = $CXX $LDFLAGS -o \$out \$in \$flags $LIBS
+    description = LINK \$in
+
+rule test
+    command = \$in && touch \$out
+    description = TEST \$in
+
+rule strip
+    command = cp -f \$in \$out && $STRIP \$out
+    description = STRIP \$in
+EOF
+
+buildlibrary() {
+    local lib
+    lib=$1
+    shift
+
+    local flags
+    flags=
+    while true; do
+        case $1 in
+            -*)
+                flags="$flags $1"
+                shift
+                ;;
+
+            *)
+                break
+        esac
+    done
+
+    local objs
+    objs=
+    for src in "$@"; do
+        local obj
+        obj="$OBJDIR/${src%%.cc}.o"
+        objs="$objs $obj"
+
+        echo build $obj : cxx $src
+        echo "    flags=$flags"
+    done
+
+    echo build $OBJDIR/$lib : library $objs
+}
+
+buildprogram() {
+    local prog
+    prog=$1
+    shift
+
+    local flags
+    flags=
+    while true; do
+        case $1 in
+            -*)
+                flags="$flags $1"
+                shift
+                ;;
+
+            *)
+                break
+        esac
+    done
+
+    local objs
+    objs=
+    for src in "$@"; do
+        objs="$objs $OBJDIR/$src"
+    done
+
+    echo build $prog : link $objs
+    echo "    flags=$flags"
+}
+
+runtest() {
+    local prog
+    prog=$1
+    shift
+
+    buildlibrary lib$prog.a \
+        "$@"
+
+    buildprogram $OBJDIR/$prog$EXTENSION \
+        lib$prog.a \
+        libbackend.a \
+        libfmt.a
+
+    echo build $OBJDIR/$prog.stamp : test $OBJDIR/$prog$EXTENSION
+}
+
+buildlibrary libfmt.a \
+    dep/fmt/format.cc \
+    dep/fmt/posix.cc \
+
+buildlibrary libbackend.a \
+    lib/aeslanier/decoder.cc \
+    lib/amiga/decoder.cc \
+    lib/apple2/decoder.cc \
+    lib/brother/decoder.cc \
+    lib/brother/encoder.cc \
+    lib/bytes.cc \
+    lib/c64/decoder.cc \
+    lib/common/crunch.c \
+    lib/crc.cc \
+    lib/dataspec.cc \
+    lib/decoders/decoders.cc \
+    lib/decoders/fluxmapreader.cc \
+    lib/decoders/fmmfm.cc \
+    lib/encoder.cc \
+    lib/f85/decoder.cc \
+    lib/fb100/decoder.cc \
+    lib/flags.cc \
+    lib/fluxmap.cc \
+    lib/fluxsink/fluxsink.cc \
+    lib/fluxsink/hardwarefluxsink.cc \
+    lib/fluxsink/sqlitefluxsink.cc \
+    lib/fluxsource/fluxsource.cc \
+    lib/fluxsource/hardwarefluxsource.cc \
+    lib/fluxsource/kryoflux.cc \
+    lib/fluxsource/sqlitefluxsource.cc \
+    lib/fluxsource/streamfluxsource.cc \
+    lib/globals.cc \
+    lib/hexdump.cc \
+    lib/ibm/decoder.cc \
+    lib/image.cc \
+    lib/macintosh/decoder.cc \
+    lib/mx/decoder.cc \
+    lib/reader.cc \
+    lib/sector.cc \
+    lib/sectorset.cc \
+    lib/sql.cc \
+    lib/usb.cc \
+    lib/victor9k/decoder.cc \
+    lib/writer.cc \
+    lib/zilogmcz/decoder.cc \
+
+buildlibrary libfrontend.a \
+    src/fe-erase.cc \
+    src/fe-inspect.cc \
+    src/fe-readadfs.cc \
+    src/fe-readaeslanier.cc \
+    src/fe-readamiga.cc \
+    src/fe-readampro.cc \
+    src/fe-readapple2.cc \
+    src/fe-readbrother.cc \
+    src/fe-readc64.cc \
+    src/fe-readdfs.cc \
+    src/fe-readf85.cc \
+    src/fe-readfb100.cc \
+    src/fe-readibm.cc \
+    src/fe-readmac.cc \
+    src/fe-readmx.cc \
+    src/fe-readvictor9k.cc \
+    src/fe-readzilogmcz.cc \
+    src/fe-rpm.cc \
+    src/fe-seek.cc \
+    src/fe-testbulktransport.cc \
+    src/fe-upgradefluxfile.cc \
+    src/fe-writebrother.cc \
+    src/fe-writeflux.cc \
+    src/fe-writetestpattern.cc \
+    src/fluxengine.cc \
+
+buildprogram fluxengine-debug$EXTENSION \
+    libfrontend.a \
+    libbackend.a \
+    libfmt.a \
+
+echo "build fluxengine$EXTENSION : strip fluxengine-debug$EXTENSION"
+
+runtest dataspec-test       tests/dataspec.cc
+runtest flags-test          tests/flags.cc
+runtest fmmfm-test          tests/fmmfm.cc
+runtest bitaccumulator-test tests/bitaccumulator.cc
+runtest kryoflux-test       tests/kryoflux.cc
+runtest compression-test    tests/compression.cc
+runtest bytes-test          tests/bytes.cc
+runtest crunch-test         tests/crunch.cc
+runtest fluxpattern-test    tests/fluxpattern.cc