Merge pull request #99 from davidgiven/extensions

Validate image extensions before reading, not after.

.appveyor.yml

@@ -0,0 +1,41 @@
+version: '{branch}.{build}'
+clone_depth: 1
+skip_tags: true
+
+environment:
+  MSYSTEM: MINGW32
+
+init:
+  - git config --global core.autocrlf input
+
+install:
+  - set PATH=c:\msys64\mingw32\bin;c:\msys64\usr\bin;c:\msys64\bin;%PATH%
+  - echo %PATH%
+  - pacman -S --noconfirm --needed make ninja mingw-w64-i686-libusb mingw-w64-i686-sqlite3 mingw-w64-i686-zlib mingw-w64-i686-gcc zip
+
+build_script:
+  - make
+  - zip -9 fluxengine.zip fluxengine.exe brother120tool.exe FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex
+
+artifacts:
+  - path: fluxengine.zip
+    name: fluxengine.zip
+
+deploy:
+  release: FluxEngine Windows client version $(APPVEYOR_BUILD_NUMBER)
+  description: >
+    This is an automatically built version of the FluxEngine Windows client
+    which is generated whenever a significant checkin has happened. It's had
+    no testing whatsoever.
+
+    To use, download it, put it somewhere, and then run it from a cmd window
+    or other command line shell.
+  provider: GitHub
+  auth_token:
+    secure: dfJjj7fWCoDUz+Ni11OcNPB/U3TNJFwNA2AsL++ChFjniUsZLlC6SDWHiL/t4FZo
+  artifact: fluxengine.zip
+  draft: false
+  prerelease: false
+  on:
+    branch: master
+

.hgignore

@@ -3,4 +3,10 @@ streams
 .*\.flux
 .*\.img
 .*\.raw
+.*\.orig
+.vscode
+remote
+FluxEngine.cydsn/CortexM3
+FluxEngine.cydsn/Generated_Source
+FluxEngine.cydsn/codegentemp
 

.travis.yml

@@ -1,4 +1,6 @@
-language: generic
+language: shell
+git:
+    depth: 1
 
 matrix:
     include:
@@ -8,10 +10,14 @@ matrix:
             dist: xenial
             compiler: gcc
             env: CXX=g++-8
+            script:
+            - make
         -
             os: osx
-            env: HOMEBREW_NO_AUTO_UPDATE=1
+            osx_image: xcode10.2
             compiler: clang
+            env:
+            - HOMEBREW_NO_INSTALL_CLEANUP=1
 
 addons:
     apt:
@@ -20,18 +26,14 @@ addons:
         - ubuntu-toolchain-r-test
         packages:
         - ninja-build
-        - meson
         - libusb-1.0-0-dev
         - libsqlite3-dev
         - g++-8
     homebrew:
         packages:
         - ninja
-        - meson
-
-git:
-    depth: 1
 
 script:
-    - make
+- make
+
 

FluxEngine.cydsn/FluxEngine.cydwr

@@ -28,13 +28,37 @@
         <Data key="sync_with_bus_clk" value="True" />
         <Data key="user_set_domain" value="False" />
       </Group>
+      <Group key="4eef02b9-8ad1-43c4-85f1-b3335faa5fc4">
+        <Data key="check_tolerance" value="True" />
+        <Data key="clock_version" value="v1" />
+        <Data key="derive_type" value="NAMED_DIVIDER" />
+        <Data key="desired_freq" value="0" />
+        <Data key="desired_unit" value="15" />
+        <Data key="divider" value="0" />
+        <Data key="domain" value="DIGITAL" />
+        <Data key="enabled" value="True" />
+        <Data key="minus_accuracy" value="0.25" />
+        <Data key="minus_tolerance" value="5" />
+        <Data key="name" value="Clock_3" />
+        <Data key="named_src_direct_connect" value="True" />
+        <Data key="netlist_name" value="Clock_3" />
+        <Data key="placement" value="AUTO" />
+        <Data key="plus_accuracy" value="0.25" />
+        <Data key="plus_tolerance" value="5" />
+        <Data key="scope" value="LOCAL" />
+        <Data key="src_clk_id" value="75C2148C-3656-4d8a-846D-0CAE99AB6FF7" />
+        <Data key="src_clk_name" value="BUS_CLK" />
+        <Data key="start_on_reset" value="True" />
+        <Data key="sync_with_bus_clk" value="True" />
+        <Data key="user_set_domain" value="False" />
+      </Group>
       <Group key="06c4d5d4-f15f-4b29-a1d0-c24b2e38b1ec">
         <Data key="check_tolerance" value="True" />
         <Data key="clock_version" value="v1" />
         <Data key="derive_type" value="NAMED_FREQ" />
-        <Data key="desired_freq" value="24" />
+        <Data key="desired_freq" value="12" />
         <Data key="desired_unit" value="6" />
-        <Data key="divider" value="1" />
+        <Data key="divider" value="2" />
         <Data key="domain" value="DIGITAL" />
         <Data key="enabled" value="True" />
         <Data key="minus_accuracy" value="0.25" />
@@ -50,7 +74,7 @@
         <Data key="src_clk_name" value="IMO" />
         <Data key="start_on_reset" value="True" />
         <Data key="sync_with_bus_clk" value="True" />
-        <Data key="user_set_domain" value="False" />
+        <Data key="user_set_domain" value="True" />
       </Group>
       <Group key="24cd38f7-f472-4403-837f-86807c8f5333">
         <Data key="check_tolerance" value="True" />
@@ -193,6 +217,54 @@
         <Data key="sync_with_bus_clk" value="True" />
         <Data key="user_set_domain" value="False" />
       </Group>
+      <Group key="75187c05-9501-4450-b306-6ccdd3bb77db">
+        <Data key="check_tolerance" value="True" />
+        <Data key="clock_version" value="v1" />
+        <Data key="derive_type" value="NAMED_DIVIDER" />
+        <Data key="desired_freq" value="0" />
+        <Data key="desired_unit" value="15" />
+        <Data key="divider" value="0" />
+        <Data key="domain" value="DIGITAL" />
+        <Data key="enabled" value="True" />
+        <Data key="minus_accuracy" value="0.25" />
+        <Data key="minus_tolerance" value="5" />
+        <Data key="name" value="Clock_5" />
+        <Data key="named_src_direct_connect" value="True" />
+        <Data key="netlist_name" value="Clock_5" />
+        <Data key="placement" value="AUTO" />
+        <Data key="plus_accuracy" value="0.25" />
+        <Data key="plus_tolerance" value="5" />
+        <Data key="scope" value="LOCAL" />
+        <Data key="src_clk_id" value="75C2148C-3656-4d8a-846D-0CAE99AB6FF7" />
+        <Data key="src_clk_name" value="BUS_CLK" />
+        <Data key="start_on_reset" value="True" />
+        <Data key="sync_with_bus_clk" value="True" />
+        <Data key="user_set_domain" value="False" />
+      </Group>
+      <Group key="b762c287-7f87-4b21-982e-84be01dc5115">
+        <Data key="check_tolerance" value="True" />
+        <Data key="clock_version" value="v1" />
+        <Data key="derive_type" value="NAMED_DIVIDER" />
+        <Data key="desired_freq" value="0" />
+        <Data key="desired_unit" value="15" />
+        <Data key="divider" value="0" />
+        <Data key="domain" value="DIGITAL" />
+        <Data key="enabled" value="True" />
+        <Data key="minus_accuracy" value="0.25" />
+        <Data key="minus_tolerance" value="5" />
+        <Data key="name" value="Clock_2" />
+        <Data key="named_src_direct_connect" value="True" />
+        <Data key="netlist_name" value="Clock_2" />
+        <Data key="placement" value="AUTO" />
+        <Data key="plus_accuracy" value="0.25" />
+        <Data key="plus_tolerance" value="5" />
+        <Data key="scope" value="LOCAL" />
+        <Data key="src_clk_id" value="75C2148C-3656-4d8a-846D-0CAE99AB6FF7" />
+        <Data key="src_clk_name" value="BUS_CLK" />
+        <Data key="start_on_reset" value="True" />
+        <Data key="sync_with_bus_clk" value="True" />
+        <Data key="user_set_domain" value="False" />
+      </Group>
       <Group key="b0162966-0060-4af5-82d1-fcb491ad7619/be0a0e37-ad17-42ca-b5a1-1a654d736358">
         <Data key="check_tolerance" value="True" />
         <Data key="clock_version" value="v1" />
@@ -530,13 +602,14 @@
     <Group key="v1">
       <Data key="cy_boot" value="cy_boot_v5_80" />
       <Data key="Em_EEPROM_Dynamic" value="Em_EEPROM_Dynamic_v2_20" />
-      <Data key="LIN_Dynamic" value="LIN_Dynamic_v4_0" />
+      <Data key="LIN_Dynamic" value="LIN_Dynamic_v5_0" />
     </Group>
   </Group>
   <Data key="DataVersionKey" value="2" />
   <Group key="DWRInstGuidMapping">
     <Group key="Clock">
       <Data key="0b2f9bbb-00ce-4115-a788-ffb9d046a9e5" value="Clock_4" />
+      <Data key="4eef02b9-8ad1-43c4-85f1-b3335faa5fc4" value="Clock_3" />
       <Data key="06c4d5d4-f15f-4b29-a1d0-c24b2e38b1ec" value="CounterClock" />
       <Data key="24cd38f7-f472-4403-837f-86807c8f5333" value="PULSE_CLOCK" />
       <Data key="63ed4137-0b09-4256-8a27-35c9a2653f1a" value="Clock_2" />
@@ -544,6 +617,8 @@
       <Data key="349ffa20-8576-4ac3-9a6f-34ef606de6cf" value="Clock_1" />
       <Data key="6616e828-6611-4893-a674-66c861d79d6c" value="SignalSamplingClock" />
       <Data key="12664fc6-9d70-44b1-8a49-887a292e1b7f" value="Clock_3" />
+      <Data key="75187c05-9501-4450-b306-6ccdd3bb77db" value="Clock_5" />
+      <Data key="b762c287-7f87-4b21-982e-84be01dc5115" value="Clock_2" />
       <Data key="b0162966-0060-4af5-82d1-fcb491ad7619/be0a0e37-ad17-42ca-b5a1-1a654d736358" value="UART_IntClock" />
       <Data key="cb7e877c-9fb4-4fc1-a708-f1e48eb5a68c" value="CounterClock" />
       <Data key="e4a53a4c-40e1-4747-a72a-10193ffdf31c" value="Clock_1" />
@@ -553,6 +628,7 @@
       <Data key="4a398466-709f-4228-9500-96178658e13e" value="RDATA" />
       <Data key="5a3407c1-b434-4438-a7b4-b9dfd2280495" value="MOTEA" />
       <Data key="8d318d8b-cf7b-4b6b-b02c-ab1c5c49d0ba" value="SW1" />
+      <Data key="12e00eac-69b5-4717-85c8-25ef6b224d4c" value="DEBUG_PINS" />
       <Data key="41e2d8ed-5494-4d8c-8ff7-f4f789cece51" value="REDWC" />
       <Data key="264be2d3-9481-494b-8d9c-c1905a45e9cc" value="FDD" />
       <Data key="472f8fdb-f772-44fb-8897-cc690694237b" value="WDATA" />
@@ -561,6 +637,7 @@
       <Data key="1425177d-0d0e-4468-8bcc-e638e5509a9b" value="UartRx" />
       <Data key="a5d987c6-e45b-45b9-ad93-656fab06d720" value="TRK00" />
       <Data key="a93ef5b3-00f4-42c0-8fad-0e275a7e2537" value="MOTEB" />
+      <Data key="b8380fb7-fdb8-449f-bd8d-c4ca96cdf55a" value="DEBUG_PINS" />
       <Data key="bc5d52a1-1b25-4aa0-9ba9-3f81d122772f" value="DEBUG_PINS" />
       <Data key="beca5e2d-f70f-4900-a4db-7eca1ed3126e/8b77a6c4-10a0-4390-971c-672353e2a49c" value="USBFS_Dm" />
       <Data key="beca5e2d-f70f-4900-a4db-7eca1ed3126e/618a72fc-5ddd-4df5-958f-a3d55102db42" value="USBFS_Dp" />
@@ -571,6 +648,7 @@
       <Data key="e51063a9-4fad-40c7-a06b-7cc4b137dc18" value="DSKCHG" />
       <Data key="ea7ee228-8b3f-426c-8bb8-cd7a81937769" value="DIR" />
       <Data key="ed092b9b-d398-4703-be89-cebf998501f6" value="UartTx" />
+      <Data key="fbd1f839-40f9-498e-a48b-5f3048ea5c3d/52f31aa9-2f0a-497d-9a1f-1424095e13e6" value="UART_tx" />
       <Data key="fede1767-f3fd-4021-b3d7-8f9d88f36f9b" value="DRVSA" />
       <Data key="fff78075-035e-43d7-8577-bc5be4d21926" value="WGATE" />
     </Group>
@@ -3674,6 +3752,20 @@
         <Data key="Port Format" value="2,2" />
       </Group>
     </Group>
+    <Group key="12e00eac-69b5-4717-85c8-25ef6b224d4c">
+      <Group key="0">
+        <Data key="Port Format" value="2,2" />
+      </Group>
+      <Group key="1">
+        <Data key="Port Format" value="2,3" />
+      </Group>
+      <Group key="2">
+        <Data key="Port Format" value="2,4" />
+      </Group>
+      <Group key="3">
+        <Data key="Port Format" value="2,0" />
+      </Group>
+    </Group>
     <Group key="41e2d8ed-5494-4d8c-8ff7-f4f789cece51">
       <Group key="0">
         <Data key="Port Format" value="2,7" />
@@ -3756,6 +3848,17 @@
         <Data key="Port Format" value="12,1" />
       </Group>
     </Group>
+    <Group key="b8380fb7-fdb8-449f-bd8d-c4ca96cdf55a">
+      <Group key="0">
+        <Data key="Port Format" value="2,5" />
+      </Group>
+      <Group key="1">
+        <Data key="Port Format" value="2,4" />
+      </Group>
+      <Group key="2">
+        <Data key="Port Format" value="2,3" />
+      </Group>
+    </Group>
     <Group key="bc5d52a1-1b25-4aa0-9ba9-3f81d122772f">
       <Group key="0">
         <Data key="Port Format" value="0,5" />
@@ -3809,6 +3912,11 @@
         <Data key="Port Format" value="12,7" />
       </Group>
     </Group>
+    <Group key="fbd1f839-40f9-498e-a48b-5f3048ea5c3d/52f31aa9-2f0a-497d-9a1f-1424095e13e6">
+      <Group key="0">
+        <Data key="Port Format" value="2,5" />
+      </Group>
+    </Group>
     <Group key="fede1767-f3fd-4021-b3d7-8f9d88f36f9b">
       <Group key="0">
         <Data key="Port Format" value="12,2" />
@@ -3834,7 +3942,7 @@
     <Data key="CYDEV_ECC_ENABLE" value="False" />
     <Data key="CYDEV_HEAP_SIZE" value="0x80" />
     <Data key="CYDEV_INSTRUCT_CACHE_ENABLED" value="True" />
-    <Data key="CYDEV_PROTECTION_ENABLE" value="False" />
+    <Data key="CYDEV_PROTECTION_ENABLE" value="True" />
     <Data key="CYDEV_STACK_SIZE" value="0x0800" />
     <Data key="CYDEV_TEMPERATURE" value="0C - 85/125C" />
     <Data key="CYDEV_TRACE_ENABLED" value="False" />

FluxEngine.cydsn/FluxEngine.cyprj

@@ -39,6 +39,20 @@
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="crunch.c" persistent="..\lib\common\crunch.c">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_C;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="crunch.h" persistent="..\lib\common\crunch.h">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="HEADER;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 </dependencies>
 </CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
@@ -1692,20 +1706,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART" persistent="">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART.c" persistent="Generated_Source\PSoC5\UART.c">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART.h" persistent="Generated_Source\PSoC5\UART.h">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -1731,6 +1745,34 @@
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmGnu.s" persistent="Generated_Source\PSoC5\UART_AsmGnu.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;b98f980c-3bd1-4fc7-a887-c56a20a46fdd;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmRv.s" persistent="Generated_Source\PSoC5\UART_AsmRv.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;fdb8e1ae-f83a-46cf-9446-1d703716f38a;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_PVT.h" persistent="Generated_Source\PSoC5\UART_PVT.h">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="HEADER;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="UART_AsmIar.s" persistent="Generated_Source\PSoC5\UART_AsmIar.s">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_ASM;CortexM0,CortexM0p,CortexM3,CortexM4,CortexM7;;e9305a93-d091-4da5-bdc7-2813049dcdbf;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 </dependencies>
 </CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
@@ -2164,20 +2206,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep4_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep4_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep4_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2190,20 +2232,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep3_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep3_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep3_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2216,20 +2258,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep2_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep2_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep2_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2242,20 +2284,20 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1" persistent="">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1_dma.c" persistent="Generated_Source\PSoC5\USBFS_ep1_dma.c">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="SOURCE_C;CortexM3;;;" />
 <PropertyDeltas />
 </CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="USBFS_ep1_dma.h" persistent="Generated_Source\PSoC5\USBFS_ep1_dma.h">
-<Hidden v="False" />
+<Hidden v="True" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -2268,13 +2310,13 @@
 <CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
 <CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="DEBUG_PINS" persistent="">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
 <dependencies>
 <CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
 <CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="DEBUG_PINS_aliases.h" persistent="Generated_Source\PSoC5\DEBUG_PINS_aliases.h">
-<Hidden v="True" />
+<Hidden v="False" />
 </CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
 <build_action v="HEADER;;;;" />
 <PropertyDeltas />
@@ -3207,6 +3249,39 @@
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
 <filters />
 </CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0>
+<CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFolderSerialize" version="3">
+<CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtBaseContainerSerialize" version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="CAPTURE_CONTROL" persistent="">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<CyGuid_0820c2e7-528d-4137-9a08-97257b946089 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemListSerialize" version="2">
+<dependencies>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="CAPTURE_CONTROL.h" persistent="Generated_Source\PSoC5\CAPTURE_CONTROL.h">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="HEADER;;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="CAPTURE_CONTROL.c" persistent="Generated_Source\PSoC5\CAPTURE_CONTROL.c">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_C;CortexM3;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+<CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtFileSerialize" version="3" xml_contents_version="1">
+<CyGuid_31768f72-0253-412b-af77-e7dba74d1330 type_name="CyDesigner.Common.ProjMgmt.Model.CyPrjMgmtItemSerialize" version="2" name="CAPTURE_CONTROL_PM.c" persistent="Generated_Source\PSoC5\CAPTURE_CONTROL_PM.c">
+<Hidden v="False" />
+</CyGuid_31768f72-0253-412b-af77-e7dba74d1330>
+<build_action v="SOURCE_C;CortexM3;;;" />
+<PropertyDeltas />
+</CyGuid_8b8ab257-35d3-4473-b57b-36315200b38b>
+</dependencies>
+</CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
+</CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
+<filters />
+</CyGuid_ebc4f06d-207f-49c2-a540-72acf4adabc0>
 </dependencies>
 </CyGuid_0820c2e7-528d-4137-9a08-97257b946089>
 </CyGuid_2f73275c-45bf-46ba-b3b1-00a2fe0c8dd8>
@@ -3428,7 +3503,7 @@
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Optimization Level" v="" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Link Time Optimization" v="" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@Linker@Optimization@SHARED Fat LTO objects" v="" />
-<name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Pre Build Commands" v="" />
+<name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Pre Build Commands" v="cscript patcher.vbs" />
 <name_val_pair name="c9323d49-d323-40b8-9b59-cc008d68a989@Release@CortexM3@User Commands@General@Post Build Commands" v="" />
 </name>
 </platform>
@@ -3594,6 +3669,14 @@
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@Linker@Command Line@Command Line" v="--semihosting" />
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@User Commands@General@Pre Build Commands" v="" />
 <name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Debug@CortexM3@User Commands@General@Post Build Commands" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@General@Output Directory" v="${ProjectDir}\${ProcessorType}\${Platform}\${Config}" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Assembly@Command Line@Command Line" v="-s+ -M&lt;&gt; -w+ -r -DNDEBUG --fpu None" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@C/C++@General@Preprocessor Definitions" v="-D NDEBUG -D CY_CORE_ID=0" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@C/C++@Command Line@Command Line" v="-D NDEBUG -D CY_CORE_ID=0 --debug --endian=little -e --fpu=None --no_wrap_diagnostics" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Library Generation@Command Line@Command Line" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@Linker@Command Line@Command Line" v="--semihosting" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@User Commands@General@Pre Build Commands" v="" />
+<name_val_pair name="e9305a93-d091-4da5-bdc7-2813049dcdbf@Release@CortexM3@User Commands@General@Post Build Commands" v="" />
 </name>
 </platform>
 </platforms>
@@ -3617,6 +3700,6 @@
 </ignored_deps>
 </CyGuid_495451fe-d201-4d01-b22d-5d3f5609ac37>
 <boot_component v="" />
-<current_generation v="52" />
+<current_generation v="68" />
 </CyGuid_fec8f9e8-2365-4bdb-96d3-a4380222e01b>
 </CyXmlSerializer>

FluxEngine.cydsn/main.c

@@ -1,13 +1,15 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdarg.h>
 #include <setjmp.h>
 #include "project.h"
 #include "../protocol.h"
+#include "../lib/common/crunch.h"
 
 #define MOTOR_ON_TIME 5000 /* milliseconds */
 #define STEP_INTERVAL_TIME 6 /* ms */
-#define STEP_SETTLING_TIME 40 /* ms */
+#define STEP_SETTLING_TIME 50 /* ms */
 
 #define DISKSTATUS_WPT    1
 #define DISKSTATUS_DSKCHG 2
@@ -22,12 +24,13 @@ static bool motor_on = false;
 static uint32_t motor_on_time = 0;
 static bool homed = false;
 static int current_track = 0;
-static int current_drive = 0;
+static uint8_t current_drive_flags = 0;
 
 #define BUFFER_COUNT 16
 #define BUFFER_SIZE 64
 static uint8_t td[BUFFER_COUNT];
 static uint8_t dma_buffer[BUFFER_COUNT][BUFFER_SIZE] __attribute__((aligned()));
+static uint8_t usb_buffer[BUFFER_SIZE] __attribute__((aligned()));
 static uint8_t dma_channel;
 #define NEXT_BUFFER(b) (((b)+1) % BUFFER_COUNT)
 
@@ -69,16 +72,17 @@ CY_ISR(replay_dma_finished_irq_cb)
         dma_underrun = true;
 }
 
-static void print(const char* s)
+static void print(const char* msg, ...)
 {
-    /* UART_PutString(s); */
-}
-
-static void printi(int i)
-{
-    char buffer[16];
-    sprintf(buffer, "%d", i);
-    print(buffer);
+    char buffer[64];
+    
+    va_list ap;
+    va_start(ap, msg);
+    vsnprintf(buffer, sizeof(buffer), msg, ap);
+    va_end(ap);
+    
+    UART_PutString(buffer);
+    UART_PutCRLF();
 }
 
 static void start_motor(void)
@@ -89,9 +93,6 @@ static void start_motor(void)
         CyDelay(1000);
         homed = false;
     }
-    
-    if (DISKSTATUS_REG_Read() & DISKSTATUS_DSKCHG)
-        homed = false;
 
     motor_on_time = clock;
     motor_on = true;
@@ -106,6 +107,7 @@ static void wait_until_writeable(int ep)
 
 static void send_reply(struct any_frame* f)
 {
+    print("reply 0x%02x", f->f.type);
     wait_until_writeable(FLUXENGINE_CMD_IN_EP_NUM);
     USBFS_LoadInEP(FLUXENGINE_CMD_IN_EP_NUM, (uint8_t*) f, f->f.size);
 }
@@ -136,8 +138,10 @@ static void cmd_get_version(struct any_frame* f)
 
 static void step(int dir)
 {
+    STEP_REG_Write(dir);
+    CyDelayUs(1);
     STEP_REG_Write(dir | 2);
-    CyDelay(1);
+    CyDelayUs(1);
     STEP_REG_Write(dir);
     CyDelay(STEP_INTERVAL_TIME);
 }
@@ -147,6 +151,7 @@ static void seek_to(int track)
     start_motor();
     if (!homed)
     {
+        print("homing");
         while (!TRACK0_REG_Read())
             step(STEP_TOWARDS0);
             
@@ -155,11 +160,19 @@ static void seek_to(int track)
         
         homed = true;
         current_track = 0;
-        CyDelay(1); /* for direction change */
+        CyDelayUs(1); /* for direction change */
     }
     
+    print("beginning seek from %d to %d", current_track, track);
     while (track != current_track)
     {
+        if (TRACK0_REG_Read())
+        {
+            if (current_track != 0)
+                print("unexpectedly detected track 0");
+            current_track = 0;
+        }
+        
         if (track > current_track)
         {
             step(STEP_AWAYFROM0);
@@ -170,8 +183,10 @@ static void seek_to(int track)
             step(STEP_TOWARDS0);
             current_track--;
         }
+        CyWdtClear();
     }
     CyDelay(STEP_SETTLING_TIME);
+    print("finished seek");
 }
 
 static void cmd_seek(struct seek_frame* f)
@@ -276,16 +291,22 @@ static void cmd_read(struct read_frame* f)
 
     /* Wait for the beginning of a rotation. */
         
+    print("wait");
     index_irq = false;
     while (!index_irq)
         ;
     index_irq = false;
     
+    crunch_state_t cs = {};
+    cs.outputptr = usb_buffer;
+    cs.outputlen = BUFFER_SIZE;
+    
     dma_writing_to_td = 0;
     dma_reading_from_td = -1;
     dma_underrun = false;
     int count = 0;
     SAMPLER_CONTROL_Write(0); /* !reset */
+    CAPTURE_CONTROL_Write(1);
     CyDmaChSetInitialTd(dma_channel, td[dma_writing_to_td]);
     CyDmaClearPendingDrq(dma_channel);
     CyDmaChEnable(dma_channel, 1);
@@ -302,6 +323,8 @@ static void cmd_read(struct read_frame* f)
     int revolutions = f->revolutions;
     while (!dma_underrun)
     {
+        CyWdtClear();
+
         /* Have we reached the index pulse? */
         if (index_irq)
         {
@@ -319,30 +342,48 @@ static void cmd_read(struct read_frame* f)
                 goto abort;
         }
 
-        while (USBFS_GetEPState(FLUXENGINE_DATA_IN_EP_NUM) != USBFS_IN_BUFFER_EMPTY)
+        uint8_t dma_buffer_usage = 0;
+        while (dma_buffer_usage < BUFFER_SIZE)
         {
-            if (index_irq || dma_underrun)
-                goto abort;
-        }
+            cs.inputptr = dma_buffer[dma_reading_from_td] + dma_buffer_usage;
+            cs.inputlen = BUFFER_SIZE - dma_buffer_usage;
+            crunch(&cs);
+            dma_buffer_usage += BUFFER_SIZE - cs.inputlen;
+            count++;
+            if (cs.outputlen == 0)
+            {
+                while (USBFS_GetEPState(FLUXENGINE_DATA_IN_EP_NUM) != USBFS_IN_BUFFER_EMPTY)
+                {
+                    if (index_irq || dma_underrun)
+                        goto abort;
+                }
 
-        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, dma_buffer[dma_reading_from_td], BUFFER_SIZE);
+                USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, usb_buffer, BUFFER_SIZE);
+                cs.outputptr = usb_buffer;
+                cs.outputlen = BUFFER_SIZE;
+            }
+        }
         dma_reading_from_td = NEXT_BUFFER(dma_reading_from_td);
-        count++;
     }
-abort:
+abort:;
+    CAPTURE_CONTROL_Write(0);
     CyDmaChSetRequest(dma_channel, CY_DMA_CPU_TERM_CHAIN);
     while (CyDmaChGetRequest(dma_channel))
         ;
 
+    donecrunch(&cs);
+    wait_until_writeable(FLUXENGINE_DATA_IN_EP_NUM);
+    unsigned zz = cs.outputlen;
+    if (cs.outputlen != BUFFER_SIZE)
+        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, usb_buffer, BUFFER_SIZE-cs.outputlen);
+    if ((cs.outputlen == BUFFER_SIZE) || (cs.outputlen == 0))
+        USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, NULL, 0);
     wait_until_writeable(FLUXENGINE_DATA_IN_EP_NUM);
-    USBFS_LoadInEP(FLUXENGINE_DATA_IN_EP_NUM, NULL, 0);
     deinit_dma();
 
     if (dma_underrun)
     {
-        print("underrun after ");
-        printi(count);
-        print(" packets\r");
+        print("underrun after %d packets");
         send_error(F_ERROR_UNDERRUN);
     }
     else
@@ -350,6 +391,7 @@ abort:
         DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_READ_REPLY);
         send_reply(&r);
     }
+    print("count=%d i=%d d=%d zz=%d", count, index_irq, dma_underrun, zz);
 }
 
 static void init_replay_dma(void)
@@ -395,32 +437,80 @@ static void cmd_write(struct write_frame* f)
 
     init_replay_dma();
     bool writing = false; /* to the disk */
-    bool listening = false;
     bool finished = false;
     int packets = f->bytes_to_write / FRAME_SIZE;
-    int count_read = 0;
     int count_written = 0;
+    int count_read = 0;
     dma_writing_to_td = 0;
     dma_reading_from_td = -1;
     dma_underrun = false;
 
+    crunch_state_t cs = {};
+    cs.outputlen = BUFFER_SIZE;
+    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
+    
     int old_reading_from_td = -1;
     for (;;)
     {
-        if (dma_reading_from_td != old_reading_from_td)
-        {
-            count_written++;
-            old_reading_from_td = dma_reading_from_td;
-        }
+        /* Read data from USB into the buffers. */
         
-        if (dma_reading_from_td != -1)
+        if (NEXT_BUFFER(dma_writing_to_td) != dma_reading_from_td)
         {
-            /* We want to be writing to disk. */
+            if (writing && (dma_underrun || index_irq))
+                goto abort;
             
-            if (!writing)
+            /* Read crunched data, if necessary. */
+            
+            if (cs.inputlen == 0)
             {
-                print("start writing\r");
+                if (finished)
+                {
+                    /* There's no more data to read, so fake some. */
+                    
+                    for (int i=0; i<BUFFER_SIZE; i++)
+                        usb_buffer[i+0] = 0x7f;
+                    cs.inputptr = usb_buffer;
+                    cs.inputlen = BUFFER_SIZE;
+                }
+                else
+                {
+                    while (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) != USBFS_OUT_BUFFER_FULL)
+                    {
+                        if (writing && (dma_underrun || index_irq))
+                            goto abort;
+                    }
 
+                    int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, usb_buffer);
+                    cs.inputptr = usb_buffer;
+                    cs.inputlen = length;
+                    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
+
+                    count_read++;
+                    if ((length < FRAME_SIZE) || (count_read == packets))
+                        finished = true;
+                }
+            }
+            
+            /* If there *is* data waiting in the buffer, uncrunch it. */
+            
+            if (cs.inputlen != 0)
+            {
+                cs.outputptr = dma_buffer[dma_writing_to_td] + BUFFER_SIZE - cs.outputlen;
+                uncrunch(&cs);
+                if (cs.outputlen == 0)
+                {
+                    /* Completed a DMA buffer; queue it for writing. */
+                    
+                    dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
+                    cs.outputlen = BUFFER_SIZE;
+                }
+            }
+            
+            /* If we have a full buffer, start writing. */
+            if ((dma_reading_from_td == -1) && (dma_writing_to_td == BUFFER_COUNT-1))
+            {
+                dma_reading_from_td = old_reading_from_td = 0;
+                
                 /* Start the DMA engine. */
                 
                 SEQUENCER_DMA_FINISHED_IRQ_Enable();
@@ -441,90 +531,38 @@ static void cmd_write(struct write_frame* f)
                 ERASE_REG_Write(1); /* start erasing! */
                 SEQUENCER_CONTROL_Write(0); /* start writing! */
             }
-            
-            /* ...unless we reach the end of the track or suffer underrung, of course. */
-            
-            if (index_irq || dma_underrun)
-                break;
         }
         
-        if (NEXT_BUFFER(dma_writing_to_td) != dma_reading_from_td)
+        if (writing && (dma_underrun || index_irq))
+            goto abort;
+
+        if (dma_reading_from_td != old_reading_from_td)
         {
-            /* We're ready for more data. */
-            
-            if (finished)
-            {
-                /* The USB stream has stopped early, so just fake data to keep the writer happy. */
-                
-                for (int i=0; i<BUFFER_SIZE; i++)
-                    dma_buffer[dma_writing_to_td][i] = 0x80;
-                dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
-            }
-            else
-            {
-                /* Make sure we're waiting for USB data. */
-                
-                if (!listening)
-                {
-                    USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
-                    listening = true;
-                }
-                         
-                /* Is more data actually ready? */
-                
-                if (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) == USBFS_OUT_BUFFER_FULL)
-                {            
-                    int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, dma_buffer[dma_writing_to_td]);
-                    listening = false;
-                    dma_writing_to_td = NEXT_BUFFER(dma_writing_to_td);
-                    
-                    count_read++;
-                    if ((length < FRAME_SIZE) || (count_read == packets))
-                        finished = true;
-                }
-            }
-            
-            /* Only start writing once the buffer is full. */
-            
-            if ((dma_reading_from_td == -1) && (dma_writing_to_td == BUFFER_COUNT-1))
-                dma_reading_from_td = old_reading_from_td = 0;
+            count_written++;
+            old_reading_from_td = dma_reading_from_td;
         }
     }
+abort:
     SEQUENCER_DMA_FINISHED_IRQ_Disable();
 
     SEQUENCER_CONTROL_Write(1); /* reset */
     if (writing)
     {
         ERASE_REG_Write(0);
-        print("stop writing after ");
-        printi(count_written);
-        print("\r");
         CyDmaChSetRequest(dma_channel, CY_DMA_CPU_TERM_CHAIN);
         while (CyDmaChGetRequest(dma_channel))
             ;
         CyDmaChDisable(dma_channel);
     }
     
-    if (dma_underrun)
-    {
-        print("underrun after ");
-        printi(count_read);
-        print(" out of ");
-        printi(packets);
-        print(" packets read\r");
-    }
-
-    DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_WRITE_REPLY);
-
+    //debug("p=%d cr=%d cw=%d f=%d l=%d w=%d index=%d underrun=%d", packets, count_read, count_written, finished, listening, writing, index_irq, dma_underrun);
     if (!finished)
     {
-        if (!listening)
-            USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
         while (count_read < packets)
         {
             if (USBFS_GetEPState(FLUXENGINE_DATA_OUT_EP_NUM) == USBFS_OUT_BUFFER_FULL)
             {
-                int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, dma_buffer[0]);
+                int length = usb_read(FLUXENGINE_DATA_OUT_EP_NUM, usb_buffer);
                 if (length < FRAME_SIZE)
                     break;
                 USBFS_EnableOutEP(FLUXENGINE_DATA_OUT_EP_NUM);
@@ -542,6 +580,7 @@ static void cmd_write(struct write_frame* f)
         return;
     }
 
+    DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_WRITE_REPLY);
     send_reply((struct any_frame*) &r);
 }
 
@@ -551,7 +590,7 @@ static void cmd_erase(struct erase_frame* f)
     seek_to(current_track);    
     /* Disk is now spinning. */
     
-    print("start erasing\r");
+    print("start erasing");
     index_irq = false;
     while (!index_irq)
         ;
@@ -560,7 +599,7 @@ static void cmd_erase(struct erase_frame* f)
     while (!index_irq)
         ;
     ERASE_REG_Write(0);
-    print("stop erasing\r");
+    print("stop erasing");
 
     DECLARE_REPLY_FRAME(struct any_frame, F_FRAME_ERASE_REPLY);
     send_reply((struct any_frame*) &r);
@@ -568,10 +607,10 @@ static void cmd_erase(struct erase_frame* f)
 
 static void cmd_set_drive(struct set_drive_frame* f)
 {
-    if (current_drive != f->drive)
+    if (current_drive_flags != f->drive_flags)
     {
-        current_drive = f->drive;
-        DRIVE_REG_Write(current_drive);
+        current_drive_flags = f->drive_flags;
+        DRIVE_REG_Write(current_drive_flags);
         homed = false;
     }
     
@@ -585,6 +624,7 @@ static void handle_command(void)
     (void) usb_read(FLUXENGINE_CMD_OUT_EP_NUM, input_buffer);
 
     struct any_frame* f = (struct any_frame*) input_buffer;
+    print("command 0x%02x", f->f.type);
     switch (f->f.type)
     {
         case F_FRAME_GET_VERSION_CMD:
@@ -637,7 +677,7 @@ int main(void)
     CAPTURE_DMA_FINISHED_IRQ_StartEx(&capture_dma_finished_irq_cb);
     SEQUENCER_DMA_FINISHED_IRQ_StartEx(&replay_dma_finished_irq_cb);
     DRIVE_REG_Write(0);
-    /* UART_Start(); */
+    UART_Start();
     USBFS_Start(0, USBFS_DWR_VDDD_OPERATION);
     
     CyWdtStart(CYWDT_1024_TICKS, CYWDT_LPMODE_DISABLED);
@@ -660,10 +700,10 @@ int main(void)
         
         if (!USBFS_GetConfiguration() || USBFS_IsConfigurationChanged())
         {
-            print("Waiting for USB...\r");
+            print("Waiting for USB...");
             while (!USBFS_GetConfiguration())
                 ;
-            print("USB ready\r");
+            print("USB ready");
             USBFS_EnableOutEP(FLUXENGINE_CMD_OUT_EP_NUM);
         }
         
@@ -671,7 +711,7 @@ int main(void)
         {
             handle_command();
             USBFS_EnableOutEP(FLUXENGINE_CMD_OUT_EP_NUM);
-            print("idle\r");
+            print("idle");
         }
     }
 }

Makefile

@@ -1,6 +1,40 @@
-all: .obj/build.ninja
-	@ninja -C .obj test
-
-.obj/build.ninja:
-	@mkdir -p .obj
-	meson .obj
+PACKAGES = zlib sqlite3 libusb-1.0
+
+export CFLAGS = -O3 -g --std=c++14 \
+	-ffunction-sections -fdata-sections
+export LDFLAGS = -O3
+
+ifeq ($(OS), Windows_NT)
+export CXX = /mingw32/bin/g++
+export AR = /mingw32/bin/ar rcs
+export STRIP = /mingw32/bin/strip
+export CFLAGS += -I/mingw32/include/libusb-1.0
+export LDFLAGS +=
+export LIBS = -static -lz -lsqlite3 -lusb-1.0
+export EXTENSION = .exe
+else
+export CXX = g++
+export AR = ar rcs
+export STRIP = strip
+export CFLAGS += $(shell pkg-config --cflags $(PACKAGES))
+export LDFLAGS +=
+export LIBS = $(shell pkg-config --libs $(PACKAGES))
+export EXTENSION =
+endif
+
+CFLAGS += -Ilib -Idep/fmt -Iarch
+
+export OBJDIR = .obj
+
+all: .obj/build.ninja
+	@ninja -f .obj/build.ninja -v
+
+clean:
+	@echo CLEAN
+	@rm -rf $(OBJDIR)
+
+.obj/build.ninja: mkninja.sh Makefile
+	@echo MKNINJA $@
+	@mkdir -p $(OBJDIR)
+	@sh $< > $@
+

README.md

@@ -29,6 +29,13 @@ the board. Sorry for the inconvenience, but it means you don't have to modify
 the board any more to make it work. If you built the hardware prior to then,
 you'll need to adjust it.
 
+**Another important note.** On 2019-07-03 I've revamped the build process and
+the (command line) user interface. It should be much nicer now, not least in
+that there's a single client binary with all the functionality in it. The
+interface is a little different, but not much. The build process is now
+better (simpler). See [the building](doc/building.md) and
+[using](doc/using.md) pages for more details.
+
 Where?
 ------
 
@@ -52,9 +59,8 @@ following friendly articles:
   - [Using a FluxEngine](doc/using.md) ∾ what to do with your new hardware ∾
     flux files and image files ∾ knowing what you're doing
 
-  - [Reading dubious disks](doc/problems.md) ∾ it's not an exact science ∾
-    the sector map ∾ clock detection and the histogram ∾ tuning the clock ∾
-    manual adjustment
+  - [Troubleshooting dubious disks](doc/problems.md) ∾ it's not an exact science ∾
+    the sector map ∾ clock detection and the histogram
 
 Which?
 ------
@@ -66,7 +72,8 @@ decoder based on Kryoflux (or other) dumps I've found. I don't (yet) have
 real, physical disks in my hand to test the capture process.
 
 Unicorns (🦄) are completely real --- this means that I've read actual,
-physical disks with these formats and so know they work.
+physical disks with these formats and so know they work (or had reports from
+people who've had it work).
 
 ### Old disk formats
 
@@ -76,12 +83,13 @@ physical disks with these formats and so know they work.
 | [Acorn ADFS](doc/disk-acornadfs.md)      |  🦄   |        | single- and double- sided           |
 | [Acorn DFS](doc/disk-acorndfs.md)        |  🦄   |        |                                     |
 | [Ampro Little Board](doc/disk-ampro.md)  |  🦖   |        |                                     |
-| [Apple II DOS 3.3](doc/disk-apple2.md)   |  🦖   |        | doesn't do logical sector remapping |
+| [Apple II DOS 3.3](doc/disk-apple2.md)   |  🦄   |        | doesn't do logical sector remapping |
 | [Amiga](doc/disk-amiga.md)               |  🦄   |        |                                     |
 | [Commodore 64 1541](doc/disk-c64.md)     |  🦖   |        | and probably the other GCR formats  |
 | [Brother 120kB](doc/disk-brother.md)     |  🦄   |        |                                     |
 | [Brother 240kB](doc/disk-brother.md)     |  🦄   |   🦄   |                                     |
-| [Macintosh 800kB](doc/disk-macintosh.md) |  🦖   |        | and probably the 400kB too          |
+| [Brother FB-100](doc/disk-fb100.md)      |  🦖   |        | Tandy Model 100, Husky Hunter, knitting machines |
+| [Macintosh 800kB](doc/disk-macintosh.md) |  🦄   |        | and probably the 400kB too          |
 | [TRS-80](doc/disk-trs80.md)              |  🦖   |        | a minor variation of the IBM scheme |
 {: .datatable }
 
@@ -97,9 +105,11 @@ at least, check the CRC so what data's there is probably good.
 |:-----------------------------------------|:-----:|:------:|-------|
 | [AES Superplus / No Problem](doc/disk-aeslanier.md) |  🦖   | | hard sectors! |
 | [Durango F85](doc/disk-durangof85.md)    |  🦖   |        | 5.25" |
+| [DVK MX](doc/disk-mx.md)                 |  🦖   |        | Soviet PDP-11 clone |
 | [Victor 9000](doc/disk-victor9k.md)      |  🦖   |        | 8-inch        |
 | [Zilog MCZ](doc/disk-zilogmcz.md)        |  🦖   |        | 8-inch _and_ hard sectors |
 {: .datatable }
+
 ### Notes
 
   - IBM PC disks are the lowest-common-denominator standard. A number of other
@@ -179,3 +189,8 @@ maintained by Victor Zverovich (`vitaut <https://github.com/vitaut>`) and
 Jonathan Müller (`foonathan <https://github.com/foonathan>`) with
 contributions from many other people. It is licensed under the terms of the
 BSD license. Please see the contents of the directory for the full text.
+
+As an exception, `dep/emu` contains parts of the OpenBSD C library
+code, Todd Miller and William A. Rowe (and probably others). It is licensed
+under the terms of the 3-clause BSD license. Please see the contents of the
+directory for the full text. It's been lightly modified by me.

arch/aeslanier/aeslanier.h

@@ -0,0 +1,20 @@
+#ifndef AESLANIER_H
+#define AESLANIER_H
+
+#define AESLANIER_RECORD_SEPARATOR 0x55555122
+#define AESLANIER_SECTOR_LENGTH    256
+#define AESLANIER_RECORD_SIZE      (AESLANIER_SECTOR_LENGTH + 5)
+
+class Sector;
+class Fluxmap;
+
+class AesLanierDecoder : public AbstractDecoder
+{
+public:
+    virtual ~AesLanierDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+};
+
+#endif

arch/aeslanier/decoder.cc

@@ -0,0 +1,65 @@
+#include "globals.h"
+#include "decoders/decoders.h"
+#include "aeslanier.h"
+#include "crc.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "sector.h"
+#include "bytes.h"
+#include "record.h"
+#include "fmt/format.h"
+#include <string.h>
+
+static const FluxPattern SECTOR_PATTERN(32, AESLANIER_RECORD_SEPARATOR);
+
+/* This is actually M2FM, rather than MFM, but it our MFM/FM decoder copes fine with it. */
+
+static Bytes reverse_bits(const Bytes& input)
+{
+    Bytes output;
+    ByteWriter bw(output);
+
+    for (uint8_t b : input)
+        bw.write_8(reverse_bits(b));
+    return output;
+}
+
+AbstractDecoder::RecordType AesLanierDecoder::advanceToNextRecord()
+{
+    _sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
+    if (_fmr->eof() || !_sector->clock)
+        return UNKNOWN_RECORD;
+    return SECTOR_RECORD;
+}
+
+void AesLanierDecoder::decodeSectorRecord()
+{
+    /* Skip ID mark. */
+
+    readRawBits(16);
+
+    const auto& rawbits = readRawBits(AESLANIER_RECORD_SIZE*16);
+    const auto& bytes = decodeFmMfm(rawbits).slice(0, AESLANIER_RECORD_SIZE);
+    const auto& reversed = reverse_bits(bytes);
+
+    _sector->logicalTrack = reversed[1];
+    _sector->logicalSide = 0;
+    _sector->logicalSector = reversed[2];
+
+    /* Check header 'checksum' (which seems far too simple to mean much). */
+
+    {
+        uint8_t wanted = reversed[3];
+        uint8_t got = reversed[1] + reversed[2];
+        if (wanted != got)
+            return;
+    }
+
+    /* Check data checksum, which also includes the header and is
+        * significantly better. */
+
+    _sector->data = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
+    uint16_t wanted = reversed.reader().seek(0x101).read_le16();
+    uint16_t got = crc16ref(MODBUS_POLY_REF, _sector->data);
+    _sector->status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/amiga/amiga.h

@@ -0,0 +1,20 @@
+#ifndef AMIGA_H
+#define AMIGA_H
+
+#define AMIGA_SECTOR_RECORD 0xaaaa44894489LL
+
+#define AMIGA_RECORD_SIZE 0x21f
+
+class Sector;
+class Fluxmap;
+
+class AmigaDecoder : public AbstractDecoder
+{
+public:
+    virtual ~AmigaDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+};
+
+#endif

arch/amiga/decoder.cc

@@ -0,0 +1,99 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "amiga.h"
+#include "bytes.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+/* 
+ * Amiga disks use MFM but it's not quite the same as IBM MFM. They only use
+ * a single type of record with a different marker byte.
+ * 
+ * See the big comment in the IBM MFM decoder for the gruesome details of how
+ * MFM works.
+ */
+         
+static const FluxPattern SECTOR_PATTERN(48, AMIGA_SECTOR_RECORD);
+
+static Bytes deinterleave(const uint8_t*& input, size_t len)
+{
+    assert(!(len & 1));
+    const uint8_t* odds = &input[0];
+    const uint8_t* evens = &input[len/2];
+    Bytes output;
+    ByteWriter bw(output);
+
+    for (size_t i=0; i<len/2; i++)
+    {
+        uint8_t o = *odds++;
+        uint8_t e = *evens++;
+
+        /* This is the 'Interleave bits with 64-bit multiply' technique from
+         * http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
+         */
+        uint16_t result =
+            (((e * 0x0101010101010101ULL & 0x8040201008040201ULL)
+                * 0x0102040810204081ULL >> 49) & 0x5555) |
+            (((o * 0x0101010101010101ULL & 0x8040201008040201ULL)
+                * 0x0102040810204081ULL >> 48) & 0xAAAA);
+        
+        bw.write_be16(result);
+    }
+
+    input += len;
+    return output;
+}
+
+static uint32_t checksum(const Bytes& bytes)
+{
+    ByteReader br(bytes);
+    uint32_t checksum = 0;
+
+    assert((bytes.size() & 3) == 0);
+    while (!br.eof())
+        checksum ^= br.read_be32();
+
+    return checksum & 0x55555555;
+}
+
+AbstractDecoder::RecordType AmigaDecoder::advanceToNextRecord()
+{
+    _sector->clock = _fmr->seekToPattern(SECTOR_PATTERN);
+    if (_fmr->eof() || !_sector->clock)
+        return UNKNOWN_RECORD;
+    return SECTOR_RECORD;
+}
+
+void AmigaDecoder::decodeSectorRecord()
+{
+    const auto& rawbits = readRawBits(AMIGA_RECORD_SIZE*16);
+    const auto& rawbytes = toBytes(rawbits).slice(0, AMIGA_RECORD_SIZE*2);
+    const auto& bytes = decodeFmMfm(rawbits).slice(0, AMIGA_RECORD_SIZE);
+
+    const uint8_t* ptr = bytes.begin() + 3;
+
+    Bytes header = deinterleave(ptr, 4);
+    Bytes recoveryinfo = deinterleave(ptr, 16);
+
+    _sector->logicalTrack = header[1] >> 1;
+    _sector->logicalSide = header[1] & 1;
+    _sector->logicalSector = header[2];
+
+    uint32_t wantedheaderchecksum = deinterleave(ptr, 4).reader().read_be32();
+    uint32_t gotheaderchecksum = checksum(rawbytes.slice(6, 40));
+    if (gotheaderchecksum != wantedheaderchecksum)
+        return;
+
+    uint32_t wanteddatachecksum = deinterleave(ptr, 4).reader().read_be32();
+    uint32_t gotdatachecksum = checksum(rawbytes.slice(62, 1024));
+
+    _sector->data.clear();
+    _sector->data.writer().append(deinterleave(ptr, 512)).append(recoveryinfo);
+    _sector->status = (gotdatachecksum == wanteddatachecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/apple2/apple2.h

@@ -10,14 +10,16 @@
 class Sector;
 class Fluxmap;
 
-class Apple2Decoder : public AbstractSoftSectorDecoder
+class Apple2Decoder : public AbstractDecoder
 {
 public:
     virtual ~Apple2Decoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
 };
 
+
 #endif
 

arch/apple2/decoder.cc

@@ -0,0 +1,112 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "apple2.h"
+#include "bytes.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_RECORD_PATTERN(24, APPLE2_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, APPLE2_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+/* This is extremely inspired by the MESS implementation, written by Nathan Woods
+ * and R. Belmont: https://github.com/mamedev/mame/blob/7914a6083a3b3a8c243ae6c3b8cb50b023f21e0e/src/lib/formats/ap2_dsk.cpp
+ */
+static Bytes decode_crazy_data(const uint8_t* inp, Sector::Status& status)
+{
+    Bytes output(APPLE2_SECTOR_LENGTH);
+
+    uint8_t checksum = 0;
+    for (unsigned i = 0; i < APPLE2_ENCODED_SECTOR_LENGTH; i++)
+    {
+        checksum ^= decode_data_gcr(*inp++);
+
+        if (i >= 86)
+        {
+            /* 6 bit */
+            output[i - 86] |= (checksum << 2);
+        }
+        else
+        {
+            /* 3 * 2 bit */
+            output[i + 0] = ((checksum >> 1) & 0x01) | ((checksum << 1) & 0x02);
+            output[i + 86] = ((checksum >> 3) & 0x01) | ((checksum >> 1) & 0x02);
+            if ((i + 172) < APPLE2_SECTOR_LENGTH)
+                output[i + 172] = ((checksum >> 5) & 0x01) | ((checksum >> 3) & 0x02);
+        }
+    }
+
+    checksum &= 0x3f;
+    uint8_t wantedchecksum = decode_data_gcr(*inp);
+    status = (checksum == wantedchecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+    return output;
+}
+
+uint8_t combine(uint16_t word)
+{
+    return word & (word >> 7);
+}
+
+AbstractDecoder::RecordType Apple2Decoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void Apple2Decoder::decodeSectorRecord()
+{
+    /* Skip ID (as we know it's a APPLE2_SECTOR_RECORD). */
+    readRawBits(24);
+
+    /* Read header. */
+
+    auto header = toBytes(readRawBits(8*8)).slice(0, 8);
+    ByteReader br(header);
+
+    uint8_t volume = combine(br.read_be16());
+    _sector->logicalTrack = combine(br.read_be16());
+    _sector->logicalSector = combine(br.read_be16());
+    uint8_t checksum = combine(br.read_be16());
+    if (checksum == (volume ^ _sector->logicalTrack ^ _sector->logicalSector))
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Apple2Decoder::decodeDataRecord()
+{
+    /* Check ID. */
+
+    Bytes bytes = toBytes(readRawBits(3*8)).slice(0, 3);
+    if (bytes.reader().read_be24() != APPLE2_DATA_RECORD)
+        return;
+
+    /* Read and decode data. */
+
+    unsigned recordLength = APPLE2_ENCODED_SECTOR_LENGTH + 2;
+    bytes = toBytes(readRawBits(recordLength*8)).slice(0, recordLength);
+
+    _sector->status = Sector::BAD_CHECKSUM;
+    _sector->data = decode_crazy_data(&bytes[0], _sector->status);
+}

arch/brother/brother.h

@@ -0,0 +1,39 @@
+#ifndef BROTHER_H
+#define BROTHER_H
+
+/* Brother word processor format (or at least, one of them) */
+
+#define BROTHER_SECTOR_RECORD            0xFFFFFD57
+#define BROTHER_DATA_RECORD              0xFFFFFDDB
+#define BROTHER_DATA_RECORD_PAYLOAD      256
+#define BROTHER_DATA_RECORD_CHECKSUM     3
+#define BROTHER_DATA_RECORD_ENCODED_SIZE 415
+
+#define BROTHER_TRACKS_PER_DISK          78
+#define BROTHER_SECTORS_PER_TRACK        12
+
+class Sector;
+class Fluxmap;
+
+class BrotherDecoder : public AbstractDecoder
+{
+public:
+    virtual ~BrotherDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+};
+
+class BrotherEncoder : public AbstractEncoder
+{
+public:
+	virtual ~BrotherEncoder() {}
+
+public:
+    std::unique_ptr<Fluxmap> encode(int physicalTrack, int physicalSide, const SectorSet& allSectors);
+};
+
+extern FlagGroup brotherEncoderFlags;
+
+#endif

arch/brother/decoder.cc

@@ -0,0 +1,109 @@
+#include "globals.h"
+#include "sql.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "decoders/decoders.h"
+#include "encoders/encoders.h"
+#include "record.h"
+#include "brother.h"
+#include "sector.h"
+#include "bytes.h"
+#include "crc.h"
+#include <ctype.h>
+
+const FluxPattern SECTOR_RECORD_PATTERN(32, BROTHER_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(32, BROTHER_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static std::vector<uint8_t> outputbuffer;
+
+/*
+ * Brother disks have this very very non-IBM system where sector header records
+ * and data records use two different kinds of GCR: sector headers are 8-in-16
+ * (but the encodable values range from 0 to 77ish only) and data headers are
+ * 5-in-8. In addition, there's a non-encoded 10-bit ID word at the beginning
+ * of each record, as well as a string of 53 1s introducing them. That does at
+ * least make them easy to find.
+ *
+ * Disk formats vary from machine to machine, but mine uses 78 tracks. Track 0
+ * is erased but not formatted.  Track alignment is extremely dubious and
+ * Brother track 0 shows up on my machine at track 2.
+ */
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+static int decode_header_gcr(uint16_t word)
+{
+	switch (word)
+	{
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "header_gcr.h"
+		#undef GCR_ENTRY
+	}                       
+	return -1;             
+};
+
+AbstractDecoder::RecordType BrotherDecoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void BrotherDecoder::decodeSectorRecord()
+{
+	readRawBits(32);
+	const auto& rawbits = readRawBits(32);
+	const auto& bytes = toBytes(rawbits).slice(0, 4);
+
+	ByteReader br(bytes);
+	_sector->logicalTrack = decode_header_gcr(br.read_be16());
+	_sector->logicalSector = decode_header_gcr(br.read_be16());
+
+	/* Sanity check the values read; there's no header checksum and
+		* occasionally we get garbage due to bit errors. */
+	if (_sector->logicalSector > 11)
+		return;
+	if (_sector->logicalTrack > 79)
+		return;
+
+	_sector->status = Sector::DATA_MISSING;
+}
+
+void BrotherDecoder::decodeDataRecord()
+{
+	readRawBits(32);
+
+	const auto& rawbits = readRawBits(BROTHER_DATA_RECORD_ENCODED_SIZE*8);
+	const auto& rawbytes = toBytes(rawbits).slice(0, BROTHER_DATA_RECORD_ENCODED_SIZE);
+
+	Bytes bytes;
+	ByteWriter bw(bytes);
+	BitWriter bitw(bw);
+	for (uint8_t b : rawbytes)
+	{
+		uint32_t nibble = decode_data_gcr(b);
+		bitw.push(nibble, 5);
+	}
+	bitw.flush();
+
+	_sector->data = bytes.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
+	uint32_t realCrc = crcbrother(_sector->data);
+	uint32_t wantCrc = bytes.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
+	_sector->status = (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/brother/encoder.cc

@@ -0,0 +1,167 @@
+#include "globals.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "encoders/encoders.h"
+#include "brother.h"
+#include "crc.h"
+#include "sectorset.h"
+#include "writer.h"
+
+FlagGroup brotherEncoderFlags;
+
+static DoubleFlag clockRateUs(
+	{ "--clock-rate" },
+	"Encoded data clock rate (microseconds).",
+	3.83);
+
+static DoubleFlag postIndexGapMs(
+	{ "--post-index-gap" },
+	"Post-index gap before first sector header (milliseconds).",
+	1.0);
+
+static DoubleFlag sectorSpacingMs(
+	{ "--sector-spacing" },
+	"Time between successive sector headers (milliseconds).",
+	16.2);
+
+static DoubleFlag postHeaderSpacingMs(
+	{ "--post-header-spacing" },
+	"Time between a sector's header and data records (milliseconds).",
+	0.69);
+
+static StringFlag sectorSkew(
+	{ "--sector-skew" },
+	"Order in which to write sectors.",
+	"05a3816b4927");
+
+static int encode_header_gcr(uint16_t word)
+{
+	switch (word)
+	{
+		#define GCR_ENTRY(gcr, data) \
+			case data: return gcr;
+		#include "header_gcr.h"
+		#undef GCR_ENTRY
+	}                       
+	return -1;             
+};
+
+static int encode_data_gcr(uint8_t data)
+{
+	switch (data)
+	{
+		#define GCR_ENTRY(gcr, data) \
+			case data: return gcr;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+	}                       
+	return -1;             
+};
+
+static void write_bits(std::vector<bool>& bits, unsigned& cursor, uint32_t data, int width)
+{
+	cursor += width;
+	for (int i=0; i<width; i++)
+	{
+		unsigned pos = cursor - i - 1;
+		if (pos < bits.size())
+			bits[pos] = data & 1;
+		data >>= 1;
+	}
+}
+
+static void write_sector_header(std::vector<bool>& bits, unsigned& cursor,
+		int track, int sector)
+{
+	write_bits(bits, cursor, 0xffffffff, 31);
+	write_bits(bits, cursor, BROTHER_SECTOR_RECORD, 32);
+	write_bits(bits, cursor, encode_header_gcr(track), 16);
+	write_bits(bits, cursor, encode_header_gcr(sector), 16);
+	write_bits(bits, cursor, encode_header_gcr(0x2f), 16);
+}
+
+static void write_sector_data(std::vector<bool>& bits, unsigned& cursor, const Bytes& data)
+{
+	write_bits(bits, cursor, 0xffffffff, 32);
+	write_bits(bits, cursor, BROTHER_DATA_RECORD, 32);
+
+	uint16_t fifo = 0;
+	int width = 0;
+
+	if (data.size() != BROTHER_DATA_RECORD_PAYLOAD)
+		Error() << "unsupported sector size";
+
+	auto write_byte = [&](uint8_t byte)
+	{
+		fifo |= (byte << (8 - width));
+		width += 8;
+
+		while (width >= 5)
+		{
+			uint8_t quintet = fifo >> 11;
+			fifo <<= 5;
+			width -= 5;
+
+			write_bits(bits, cursor, encode_data_gcr(quintet), 8);
+		}
+	};
+
+	for (uint8_t byte : data)
+		write_byte(byte);
+
+	uint32_t realCrc = crcbrother(data);
+	write_byte(realCrc>>16);
+	write_byte(realCrc>>8);
+	write_byte(realCrc);
+	write_byte(0x58); /* magic */
+    write_byte(0xd4);
+    while (width != 0)
+        write_byte(0);
+}
+
+static int charToInt(char c)
+{
+	if (isdigit(c))
+		return c - '0';
+	return 10 + tolower(c) - 'a';
+}
+
+std::unique_ptr<Fluxmap> BrotherEncoder::encode(
+	int physicalTrack, int physicalSide, const SectorSet& allSectors)
+{
+	if ((physicalTrack < 0) || (physicalTrack >= BROTHER_TRACKS_PER_DISK)
+        || (physicalSide != 0))
+		return std::unique_ptr<Fluxmap>();
+
+	int bitsPerRevolution = 200000.0 / clockRateUs;
+	const std::string& skew = sectorSkew.get();
+	std::vector<bool> bits(bitsPerRevolution);
+	unsigned cursor = 0;
+
+	for (int sectorCount=0; sectorCount<BROTHER_SECTORS_PER_TRACK; sectorCount++)
+	{
+		int sectorId = charToInt(skew.at(sectorCount));
+		double headerMs = postIndexGapMs + sectorCount*sectorSpacingMs;
+		unsigned headerCursor = headerMs*1e3 / clockRateUs;
+		double dataMs = headerMs + postHeaderSpacingMs;
+		unsigned dataCursor = dataMs*1e3 / clockRateUs;
+
+		const auto& sectorData = allSectors.get(physicalTrack, 0, sectorId);
+
+		fillBitmapTo(bits, cursor, headerCursor, { true, false });
+		write_sector_header(bits, cursor, physicalTrack, sectorId);
+		fillBitmapTo(bits, cursor, dataCursor, { true, false });
+		write_sector_data(bits, cursor, sectorData->data);
+	}
+
+	if (cursor > bits.size())
+		Error() << "track data overrun";
+	fillBitmapTo(bits, cursor, bits.size(), { true, false });
+
+	// The pre-index gap is not normally reported.
+	// std::cerr << "pre-index gap " << 200.0 - (double)cursor*clockRateUs/1e3 << std::endl;
+	
+	std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+	fluxmap->appendBits(bits, clockRateUs*1e3);
+	return fluxmap;
+}

arch/c64/c64.h

@@ -0,0 +1,21 @@
+#ifndef C64_H
+#define C64_H
+
+#define C64_SECTOR_RECORD    0xffd49
+#define C64_DATA_RECORD      0xffd57
+#define C64_SECTOR_LENGTH    256
+
+class Sector;
+class Fluxmap;
+
+class Commodore64Decoder : public AbstractDecoder
+{
+public:
+    virtual ~Commodore64Decoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+};
+
+#endif

arch/c64/decoder.cc

@@ -0,0 +1,92 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "c64.h"
+#include "crc.h"
+#include "bytes.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_RECORD_PATTERN(20, C64_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(20, C64_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+static Bytes decode(const std::vector<bool>& bits)
+{
+    Bytes output;
+    ByteWriter bw(output);
+    BitWriter bitw(bw);
+
+    auto ii = bits.begin();
+    while (ii != bits.end())
+    {
+        uint8_t inputfifo = 0;
+        for (size_t i=0; i<5; i++)
+        {
+            if (ii == bits.end())
+                break;
+            inputfifo = (inputfifo<<1) | *ii++;
+        }
+
+        bitw.push(decode_data_gcr(inputfifo), 4);
+    }
+    bitw.flush();
+
+    return output;
+}
+
+AbstractDecoder::RecordType Commodore64Decoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void Commodore64Decoder::decodeSectorRecord()
+{
+    readRawBits(20);
+
+    const auto& bits = readRawBits(5*10);
+    const auto& bytes = decode(bits).slice(0, 5);
+
+    uint8_t checksum = bytes[0];
+    _sector->logicalSector = bytes[1];
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = bytes[2] - 1;
+    if (checksum == xorBytes(bytes.slice(1, 4)))
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Commodore64Decoder::decodeDataRecord()
+{
+    readRawBits(20);
+
+    const auto& bits = readRawBits(259*10);
+    const auto& bytes = decode(bits).slice(0, 259);
+
+    _sector->data = bytes.slice(0, C64_SECTOR_LENGTH);
+    uint8_t gotChecksum = xorBytes(_sector->data);
+    uint8_t wantChecksum = bytes[256];
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/f85/decoder.cc

@@ -0,0 +1,100 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "f85.h"
+#include "crc.h"
+#include "bytes.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_RECORD_PATTERN(24, F85_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, F85_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+static Bytes decode(const std::vector<bool>& bits)
+{
+    Bytes output;
+    ByteWriter bw(output);
+    BitWriter bitw(bw);
+
+    auto ii = bits.begin();
+    while (ii != bits.end())
+    {
+        uint8_t inputfifo = 0;
+        for (size_t i=0; i<5; i++)
+        {
+            if (ii == bits.end())
+                break;
+            inputfifo = (inputfifo<<1) | *ii++;
+        }
+
+        bitw.push(decode_data_gcr(inputfifo), 4);
+    }
+    bitw.flush();
+
+    return output;
+}
+
+AbstractDecoder::RecordType DurangoF85Decoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return RecordType::DATA_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void DurangoF85Decoder::decodeSectorRecord()
+{
+    /* Skip sync bits and ID byte. */
+
+    readRawBits(24);
+
+    /* Read header. */
+
+    const auto& bytes = decode(readRawBits(6*10));
+
+    _sector->logicalSector = bytes[2];
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = bytes[0];
+
+    uint16_t wantChecksum = bytes.reader().seek(4).read_be16();
+    uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(0, 4));
+    if (wantChecksum == gotChecksum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void DurangoF85Decoder::decodeDataRecord()
+{
+    /* Skip sync bits ID byte. */
+
+    readRawBits(24);
+
+    const auto& bytes = decode(readRawBits((F85_SECTOR_LENGTH+3)*10))
+        .slice(0, F85_SECTOR_LENGTH+3);
+    ByteReader br(bytes);
+
+    _sector->data = br.read(F85_SECTOR_LENGTH);
+    uint16_t wantChecksum = br.read_be16();
+    uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, _sector->data);
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/f85/f85.h

@@ -0,0 +1,21 @@
+#ifndef F85_H
+#define F85_H
+
+#define F85_SECTOR_RECORD 0xffffce /* 1111 1111 1111 1111 1100 1110 */
+#define F85_DATA_RECORD 0xffffcb /* 1111 1111 1111 1111 1100 1101 */
+#define F85_SECTOR_LENGTH    512
+
+class Sector;
+class Fluxmap;
+
+class DurangoF85Decoder : public AbstractDecoder
+{
+public:
+    virtual ~DurangoF85Decoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+};
+
+#endif

arch/fb100/decoder.cc

@@ -0,0 +1,135 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "fb100.h"
+#include "crc.h"
+#include "bytes.h"
+#include "decoders/rawbits.h"
+#include "track.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_ID_PATTERN(16, 0xabaa);
+
+/* 
+ * Reverse engineered from a dump of the floppy drive's ROM. I have no idea how
+ * it works.
+ * 
+ * LF8BA:
+ *         clra
+ *         staa    X00B0
+ *         staa    X00B1
+ *         ldx     #$8000
+ * LF8C2:  ldaa    $00,x
+ *         inx
+ *         bsr     LF8CF
+ *         cpx     #$8011
+ *         bne     LF8C2
+ *         ldd     X00B0
+ *         rts
+ * LF8CF:
+ *         eora    X00B0
+ *         staa    X00CF
+ *         asla
+ *         asla
+ *         asla
+ *         asla
+ *         eora    X00CF
+ *         staa    X00CF
+ *         rola
+ *         rola
+ *         rola
+ *         tab
+ *         anda    #$F8
+ *         eora    X00B1
+ *         staa    X00B0
+ *         rolb
+ *         rolb
+ *         andb    #$0F
+ *         eorb    X00B0
+ *         stab    X00B0
+ *         rolb
+ *         eorb    X00CF
+ *         stab    X00B1
+ *         rts
+ */
+
+static void rol(uint8_t& b, bool& c)
+{
+    bool newc = b & 0x80;
+    b <<= 1;
+    b |= c;
+    c = newc;
+}
+
+static uint16_t checksum(const Bytes& bytes)
+{
+    uint8_t crclo = 0;
+    uint8_t crchi = 0;
+    for (uint8_t a : bytes)
+    {
+        a ^= crchi;
+        uint8_t t1 = a;
+        a <<= 4;
+        bool c = a & 0x10;
+        a ^= t1;
+        t1 = a;
+        rol(a, c);
+        rol(a, c);
+        rol(a, c);
+        uint8_t b = a;
+        a &= 0xf8;
+        a ^= crclo;
+        crchi = a;
+        rol(b, c);
+        rol(b, c);
+        b &= 0x0f;
+        b ^= crchi;
+        crchi = b;
+        rol(b, c);
+        b ^= t1;
+        crclo = b;
+    }
+
+    return (crchi << 8) | crclo;
+}
+
+AbstractDecoder::RecordType Fb100Decoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(SECTOR_ID_PATTERN, matcher);
+    if (matcher == &SECTOR_ID_PATTERN)
+		return RecordType::SECTOR_RECORD;
+	return RecordType::UNKNOWN_RECORD;
+}
+
+void Fb100Decoder::decodeSectorRecord()
+{
+    auto rawbits = readRawBits(FB100_RECORD_SIZE*16);
+
+    const Bytes bytes = decodeFmMfm(rawbits).slice(0, FB100_RECORD_SIZE);
+    ByteReader br(bytes);
+    br.seek(1);
+    const Bytes id = br.read(FB100_ID_SIZE);
+    uint16_t wantIdCrc = br.read_be16();
+    uint16_t gotIdCrc = checksum(id);
+    const Bytes payload = br.read(FB100_PAYLOAD_SIZE);
+    uint16_t wantPayloadCrc = br.read_be16();
+    uint16_t gotPayloadCrc = checksum(payload);
+
+    if (wantIdCrc != gotIdCrc)
+        return;
+
+    uint8_t abssector = id[2];
+    _sector->logicalTrack = abssector >> 1;
+    _sector->logicalSide = 0;
+    _sector->logicalSector = abssector & 1;
+    _sector->data.writer().append(id.slice(5, 12)).append(payload);
+
+    _sector->status = (wantPayloadCrc == gotPayloadCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/fb100/fb100.h

@@ -0,0 +1,22 @@
+#ifndef FB100_H
+#define FB100_H
+
+#define FB100_RECORD_SIZE 0x516 /* bytes */
+#define FB100_ID_SIZE 17
+#define FB100_PAYLOAD_SIZE 0x500
+
+class Sector;
+class Fluxmap;
+class Track;
+
+class Fb100Decoder : public AbstractDecoder
+{
+public:
+    virtual ~Fb100Decoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+};
+
+#endif
+

arch/ibm/decoder.cc

@@ -0,0 +1,153 @@
+#include "globals.h"
+#include "decoders/decoders.h"
+#include "ibm.h"
+#include "crc.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "sector.h"
+#include "record.h"
+#include <string.h>
+
+static_assert(std::is_trivially_copyable<IbmIdam>::value,
+		"IbmIdam is not trivially copyable");
+
+/*
+ * The markers at the beginning of records are special, and have
+ * missing clock pulses, allowing them to be found by the logic.
+ * 
+ * IAM record:
+ * flux:   XXXX-XXX-XXXX-X- = 0xf77a
+ * clock:  X X - X - X X X  = 0xd7
+ * data:    X X X X X X - - = 0xfc
+ * 
+ * (We just ignore this one --- it's useless and optional.)
+ */
+
+/* 
+ * IDAM record:
+ * flux:   XXXX-X-X-XXXXXX- = 0xf57e
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X X X - = 0xfe
+ */
+const FluxPattern FM_IDAM_PATTERN(16, 0xf57e);
+
+/* 
+ * DAM1 record:
+ * flux:   XXXX-X-X-XX-X-X- = 0xf56a
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - - - = 0xf8
+ */
+const FluxPattern FM_DAM1_PATTERN(16, 0xf56a);
+
+/* 
+ * DAM2 record:
+ * flux:   XXXX-X-X-XX-XXXX = 0xf56f
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - X X = 0xfb
+ */
+const FluxPattern FM_DAM2_PATTERN(16, 0xf56f);
+
+/* 
+ * TRS80DAM1 record:
+ * flux:   XXXX-X-X-XX-X-XX = 0xf56b
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - - X = 0xf9
+ */
+const FluxPattern FM_TRS80DAM1_PATTERN(16, 0xf56b);
+
+/* 
+ * TRS80DAM2 record:
+ * flux:   XXXX-X-X-XX-XXX- = 0xf56c
+ * clock:  X X - - - X X X  = 0xc7
+ * data:    X X X X X - X - = 0xfa
+ */
+const FluxPattern FM_TRS80DAM2_PATTERN(16, 0xf56c);
+
+/* MFM record separator:
+ * 0xA1 is:
+ * data:    1  0  1  0  0  0  0  1  = 0xa1
+ * mfm:     01 00 01 00 10 10 10 01 = 0x44a9
+ * special: 01 00 01 00 10 00 10 01 = 0x4489
+ *                       ^^^^^
+ * When shifted out of phase, the special 0xa1 byte becomes an illegal
+ * encoding (you can't do 10 00). So this can't be spoofed by user data.
+ * 
+ * shifted: 10 00 10 01 00 01 00 1
+ */
+const FluxPattern MFM_PATTERN(16, 0x4489);
+
+const FluxMatchers ANY_RECORD_PATTERN(
+    {
+        &MFM_PATTERN,
+        &FM_IDAM_PATTERN,
+        &FM_DAM1_PATTERN,
+        &FM_DAM2_PATTERN,
+        &FM_TRS80DAM1_PATTERN,
+        &FM_TRS80DAM2_PATTERN,
+    }
+);
+
+AbstractDecoder::RecordType IbmDecoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+
+    /* If this is the MFM prefix byte, the the decoder is going to expect three
+     * extra bytes on the front of the header. */
+    _currentHeaderLength = (matcher == &MFM_PATTERN) ? 3 : 0;
+
+    Fluxmap::Position here = tell();
+    if (_currentHeaderLength > 0)
+        readRawBits(_currentHeaderLength*16);
+    auto idbits = readRawBits(16);
+    uint8_t id = decodeFmMfm(idbits).slice(0, 1)[0];
+    seek(here);
+    
+    switch (id)
+    {
+        case IBM_IDAM:
+            return RecordType::SECTOR_RECORD;
+
+        case IBM_DAM1:
+        case IBM_DAM2:
+        case IBM_TRS80DAM1:
+        case IBM_TRS80DAM2:
+            return RecordType::DATA_RECORD;
+    }
+    return RecordType::UNKNOWN_RECORD;
+}
+
+void IbmDecoder::decodeSectorRecord()
+{
+    unsigned recordSize = _currentHeaderLength + IBM_IDAM_LEN;
+    auto bits = readRawBits(recordSize*16);
+    auto bytes = decodeFmMfm(bits).slice(0, recordSize);
+
+    ByteReader br(bytes);
+    br.seek(_currentHeaderLength);
+    br.read_8(); /* skip ID byte */
+    _sector->logicalTrack = br.read_8();
+    _sector->logicalSide = br.read_8();
+    _sector->logicalSector = br.read_8() - _sectorBase;
+    _currentSectorSize = 1 << (br.read_8() + 7);
+    uint16_t wantCrc = br.read_be16();
+    uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, _currentHeaderLength + 5));
+    if (wantCrc == gotCrc)
+        _sector->status = Sector::DATA_MISSING; /* correct but unintuitive */
+}
+
+void IbmDecoder::decodeDataRecord()
+{
+    unsigned recordLength = _currentHeaderLength + _currentSectorSize + 3;
+    auto bits = readRawBits(recordLength*16);
+    auto bytes = decodeFmMfm(bits).slice(0, recordLength);
+
+    ByteReader br(bytes);
+    br.seek(_currentHeaderLength);
+    br.read_8(); /* skip ID byte */
+
+    _sector->data = br.read(_currentSectorSize);
+    uint16_t wantCrc = br.read_be16();
+    uint16_t gotCrc = crc16(CCITT_POLY, bytes.slice(0, recordLength-2));
+    _sector->status = (wantCrc == gotCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/ibm/ibm.h

@@ -1,10 +1,11 @@
 #ifndef IBM_H
 #define IBM_H
 
-#include "decoders.h"
+#include "decoders/decoders.h"
 
 /* IBM format (i.e. ordinary PC floppies). */
 
+#define IBM_MFM_SYNC   0xA1   /* sync byte for MFM */
 #define IBM_IAM        0xFC   /* start-of-track record */
 #define IBM_IAM_LEN    1      /* plus prologue */
 #define IBM_IDAM       0xFE   /* sector header */
@@ -27,6 +28,24 @@ struct IbmIdam
     uint8_t crc[2];
 };
 
+class IbmDecoder : public AbstractDecoder
+{
+public:
+    IbmDecoder(unsigned sectorBase):
+        _sectorBase(sectorBase)
+    {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+
+private:
+    unsigned _sectorBase;
+    unsigned _currentSectorSize;
+    unsigned _currentHeaderLength;
+};
+
+#if 0
 class AbstractIbmDecoder : public AbstractSoftSectorDecoder
 {
 public:
@@ -35,7 +54,7 @@ public:
     {}
     virtual ~AbstractIbmDecoder() {}
 
-    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack);
+    SectorVector decodeToSectors(const RawRecordVector& rawRecords, unsigned physicalTrack, unsigned physicalSide);
 
 protected:
     virtual int skipHeaderBytes() const = 0;
@@ -65,12 +84,13 @@ public:
         AbstractIbmDecoder(sectorIdBase)
     {}
 
-    nanoseconds_t guessClock(Fluxmap& fluxmap, unsigned physicalTrack) const;
+    nanoseconds_t guessClock(Fluxmap& fluxmap) const;
     int recordMatcher(uint64_t fifo) const;
 
 protected:
     int skipHeaderBytes() const
     { return 3; }
 };
+#endif
 
 #endif

arch/macintosh/decoder.cc

@@ -0,0 +1,186 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "track.h"
+#include "macintosh.h"
+#include "bytes.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_RECORD_PATTERN(24, MAC_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(24, MAC_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+/* This is extremely inspired by the MESS implementation, written by Nathan Woods
+ * and R. Belmont: https://github.com/mamedev/mame/blob/4263a71e64377db11392c458b580c5ae83556bc7/src/lib/formats/ap_dsk35.cpp
+ */
+static Bytes decode_crazy_data(const Bytes& input, Sector::Status& status)
+{
+    Bytes output;
+    ByteWriter bw(output);
+    ByteReader br(input);
+
+    static const int LOOKUP_LEN = MAC_SECTOR_LENGTH / 3;
+
+    uint8_t b1[LOOKUP_LEN + 1];
+    uint8_t b2[LOOKUP_LEN + 1];
+    uint8_t b3[LOOKUP_LEN + 1];
+
+    for (int i=0; i<=LOOKUP_LEN; i++)
+    {
+        uint8_t w4 = br.read_8();
+        uint8_t w1 = br.read_8();
+        uint8_t w2 = br.read_8();
+        uint8_t w3 = (i != 174) ? br.read_8() : 0;
+
+        b1[i] = (w1 & 0x3F) | ((w4 << 2) & 0xC0);
+        b2[i] = (w2 & 0x3F) | ((w4 << 4) & 0xC0);
+        b3[i] = (w3 & 0x3F) | ((w4 << 6) & 0xC0);
+    }
+
+    /* Copy from the user's buffer to our buffer, while computing
+     * the three-byte data checksum. */
+
+    uint32_t c1 = 0;
+    uint32_t c2 = 0;
+    uint32_t c3 = 0;
+    unsigned count = 0;
+    for (;;)
+    {
+        c1 = (c1 & 0xFF) << 1;
+        if (c1 & 0x0100)
+            c1++;
+
+        uint8_t val = b1[count] ^ c1;
+        c3 += val;
+        if (c1 & 0x0100)
+        {
+            c3++;
+            c1 &= 0xFF;
+        }
+        bw.write_8(val);
+
+        val = b2[count] ^ c3;
+        c2 += val;
+        if (c3 > 0xFF)
+        {
+            c2++;
+            c3 &= 0xFF;
+        }
+        bw.write_8(val);
+
+        if (output.size() == 524)
+            break;
+
+        val = b3[count] ^ c2;
+        c1 += val;
+        if (c2 > 0xFF)
+        {
+            c1++;
+            c2 &= 0xFF;
+        }
+        bw.write_8(val);
+        count++;
+    }
+
+    uint8_t c4 = ((c1 & 0xC0) >> 6) | ((c2 & 0xC0) >> 4) | ((c3 & 0xC0) >> 2);
+    c1 &= 0x3f;
+    c2 &= 0x3f;
+    c3 &= 0x3f;
+    c4 &= 0x3f;
+    uint8_t g4 = br.read_8();
+    uint8_t g3 = br.read_8();
+    uint8_t g2 = br.read_8();
+    uint8_t g1 = br.read_8();
+    if ((g4 == c4) && (g3 == c3) && (g2 == c2) && (g1 == c1))
+        status = Sector::OK;
+
+    return output;
+}
+
+uint8_t decode_side(uint8_t side)
+{
+    /* Mac disks, being weird, use the side byte to encode both the side (in
+     * bit 5) and also whether we're above track 0x3f (in bit 6).
+     */
+
+    return !!(side & 0x40);
+}
+
+AbstractDecoder::RecordType MacintoshDecoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return DATA_RECORD;
+	return UNKNOWN_RECORD;
+}
+
+void MacintoshDecoder::decodeSectorRecord()
+{
+    /* Skip ID (as we know it's a MAC_SECTOR_RECORD). */
+    readRawBits(24);
+
+    /* Read header. */
+
+    auto header = toBytes(readRawBits(7*8)).slice(0, 7);
+                
+    uint8_t encodedTrack = decode_data_gcr(header[0]);
+    if (encodedTrack != (_track->physicalTrack & 0x3f))
+        return;
+                
+    uint8_t encodedSector = decode_data_gcr(header[1]);
+    uint8_t encodedSide = decode_data_gcr(header[2]);
+    uint8_t formatByte = decode_data_gcr(header[3]);
+    uint8_t wantedsum = decode_data_gcr(header[4]);
+
+    if (encodedSector > 11)
+        return;
+
+    _sector->logicalTrack = _track->physicalTrack;
+    _sector->logicalSide = decode_side(encodedSide);
+    _sector->logicalSector = encodedSector;
+    uint8_t gotsum = (encodedTrack ^ encodedSector ^ encodedSide ^ formatByte) & 0x3f;
+    if (wantedsum == gotsum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void MacintoshDecoder::decodeDataRecord()
+{
+    auto id = toBytes(readRawBits(24)).reader().read_be24();
+    if (id != MAC_DATA_RECORD)
+        return;
+
+    /* Read data. */
+
+    readRawBits(8); /* skip spare byte */
+    auto inputbuffer = toBytes(readRawBits(MAC_ENCODED_SECTOR_LENGTH*8))
+        .slice(0, MAC_ENCODED_SECTOR_LENGTH);
+
+    for (unsigned i=0; i<inputbuffer.size(); i++)
+        inputbuffer[i] = decode_data_gcr(inputbuffer[i]);
+        
+    _sector->status = Sector::BAD_CHECKSUM;
+    Bytes userData = decode_crazy_data(inputbuffer, _sector->status);
+    _sector->data.clear();
+    _sector->data.writer().append(userData.slice(12, 512)).append(userData.slice(0, 12));
+}

arch/macintosh/macintosh.h

@@ -0,0 +1,24 @@
+#ifndef MACINTOSH_H
+#define MACINTOSH_H
+
+#define MAC_SECTOR_RECORD   0xd5aa96 /* 1101 0101 1010 1010 1001 0110 */
+#define MAC_DATA_RECORD     0xd5aaad /* 1101 0101 1010 1010 1010 1101 */
+
+#define MAC_SECTOR_LENGTH   524 /* yes, really */
+#define MAC_ENCODED_SECTOR_LENGTH 703
+
+class Sector;
+class Fluxmap;
+
+class MacintoshDecoder : public AbstractDecoder
+{
+public:
+    virtual ~MacintoshDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+};
+
+#endif
+

arch/mx/decoder.cc

@@ -0,0 +1,75 @@
+#include "globals.h"
+#include "decoders/decoders.h"
+#include "mx/mx.h"
+#include "crc.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "sector.h"
+#include "record.h"
+#include "track.h"
+#include <string.h>
+
+const int SECTOR_SIZE = 256;
+
+/*
+ * MX disks are a bunch of sectors glued together with no gaps or sync markers,
+ * following a single beginning-of-track synchronisation and identification
+ * sequence.
+ */
+
+/* FM beginning of track marker:
+ * 1010 1010 1010 1010 1111 1111 1010 1111
+ *    a    a    a    a    f    f    a    f
+ */
+const FluxPattern ID_PATTERN(32, 0xaaaaffaf);
+
+void MxDecoder::beginTrack()
+{
+    _currentSector = -1;
+    _clock = 0;
+}
+
+AbstractDecoder::RecordType MxDecoder::advanceToNextRecord()
+{
+    if (_currentSector == -1)
+    {
+        /* First sector in the track: look for the sync marker. */
+        const FluxMatcher* matcher = nullptr;
+        _sector->clock = _clock = _fmr->seekToPattern(ID_PATTERN, matcher);
+        readRawBits(32); /* skip the ID mark */
+        _logicalTrack = decodeFmMfm(readRawBits(32)).reader().read_be16();
+    }
+    else if (_currentSector == 10)
+    {
+        /* That was the last sector on the disk. */
+        return UNKNOWN_RECORD;
+    }
+    else
+    {
+        /* Otherwise we assume the clock from the first sector is still valid.
+         * The decoder framwork will automatically stop when we hit the end of
+         * the track. */
+        _sector->clock = _clock;
+    }
+
+    _currentSector++;
+    return SECTOR_RECORD;
+}
+
+void MxDecoder::decodeSectorRecord()
+{
+    auto bits = readRawBits((SECTOR_SIZE+2)*16);
+    auto bytes = decodeFmMfm(bits).slice(0, SECTOR_SIZE+2).swab();
+
+    uint16_t gotChecksum = 0;
+    ByteReader br(bytes);
+    for (int i=0; i<(SECTOR_SIZE/2); i++)
+        gotChecksum += br.read_le16();
+    uint16_t wantChecksum = br.read_le16();
+
+    _sector->logicalTrack = _logicalTrack;
+    _sector->logicalSide = _track->physicalSide;
+    _sector->logicalSector = _currentSector;
+    _sector->data = bytes.slice(0, SECTOR_SIZE);
+    _sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/mx/mx.h

@@ -0,0 +1,21 @@
+#ifndef MX_H
+#define MX_H
+
+#include "decoders/decoders.h"
+
+class MxDecoder : public AbstractDecoder
+{
+public:
+    virtual ~MxDecoder() {}
+
+    void beginTrack();
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+
+private:
+    nanoseconds_t _clock;
+    int _currentSector;
+    int _logicalTrack;
+};
+
+#endif

arch/victor9k/decoder.cc

@@ -0,0 +1,111 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "victor9k.h"
+#include "crc.h"
+#include "bytes.h"
+#include "track.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+const FluxPattern SECTOR_RECORD_PATTERN(32, VICTOR9K_SECTOR_RECORD);
+const FluxPattern DATA_RECORD_PATTERN(32, VICTOR9K_DATA_RECORD);
+const FluxMatchers ANY_RECORD_PATTERN({ &SECTOR_RECORD_PATTERN, &DATA_RECORD_PATTERN });
+
+static int decode_data_gcr(uint8_t gcr)
+{
+    switch (gcr)
+    {
+		#define GCR_ENTRY(gcr, data) \
+			case gcr: return data;
+		#include "data_gcr.h"
+		#undef GCR_ENTRY
+    }
+    return -1;
+};
+
+static Bytes decode(const std::vector<bool>& bits)
+{
+    Bytes output;
+    ByteWriter bw(output);
+    BitWriter bitw(bw);
+
+    auto ii = bits.begin();
+    while (ii != bits.end())
+    {
+        uint8_t inputfifo = 0;
+        for (size_t i=0; i<5; i++)
+        {
+            if (ii == bits.end())
+                break;
+            inputfifo = (inputfifo<<1) | *ii++;
+        }
+
+        uint8_t decoded = decode_data_gcr(inputfifo);
+        bitw.push(decoded, 4);
+    }
+    bitw.flush();
+
+    return output;
+}
+
+AbstractDecoder::RecordType Victor9kDecoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+	_sector->clock = _fmr->seekToPattern(ANY_RECORD_PATTERN, matcher);
+	if (matcher == &SECTOR_RECORD_PATTERN)
+		return SECTOR_RECORD;
+	if (matcher == &DATA_RECORD_PATTERN)
+		return DATA_RECORD;
+	return UNKNOWN_RECORD;
+}
+
+void Victor9kDecoder::decodeSectorRecord()
+{
+    /* Skip the sync marker bit. */
+    readRawBits(23);
+
+    /* Read header. */
+
+    auto bytes = decode(readRawBits(4*10)).slice(0, 4);
+
+    uint8_t rawTrack = bytes[1];
+    _sector->logicalSector = bytes[2];
+    uint8_t gotChecksum = bytes[3];
+
+    _sector->logicalTrack = rawTrack & 0x7f;
+    _sector->logicalSide = rawTrack >> 7;
+    uint8_t wantChecksum = bytes[1] + bytes[2];
+    if ((_sector->logicalSector > 20) || (_sector->logicalTrack > 85) || (_sector->logicalSide > 1))
+        return;
+                
+    if (wantChecksum == gotChecksum)
+        _sector->status = Sector::DATA_MISSING; /* unintuitive but correct */
+}
+
+void Victor9kDecoder::decodeDataRecord()
+{
+    /* Skip the sync marker bit. */
+    readRawBits(23);
+
+    /* Read data. */
+
+    auto bytes = decode(readRawBits((VICTOR9K_SECTOR_LENGTH+5)*10))
+        .slice(0, VICTOR9K_SECTOR_LENGTH+5);
+    ByteReader br(bytes);
+
+    /* Check that this is actually a data record. */
+    
+    if (br.read_8() != 8)
+        return;
+
+    _sector->data = br.read(VICTOR9K_SECTOR_LENGTH);
+    uint16_t gotChecksum = sumBytes(_sector->data);
+    uint16_t wantChecksum = br.read_le16();
+    _sector->status = (gotChecksum == wantChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/victor9k/victor9k.h

@@ -0,0 +1,22 @@
+#ifndef VICTOR9K_H
+#define VICTOR9K_H
+
+#define VICTOR9K_SECTOR_RECORD 0xfffffeab
+#define VICTOR9K_DATA_RECORD   0xfffffea4
+
+#define VICTOR9K_SECTOR_LENGTH 512
+
+class Sector;
+class Fluxmap;
+
+class Victor9kDecoder : public AbstractDecoder
+{
+public:
+    virtual ~Victor9kDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+    void decodeDataRecord();
+};
+
+#endif

arch/zilogmcz/decoder.cc

@@ -0,0 +1,48 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "protocol.h"
+#include "record.h"
+#include "decoders/decoders.h"
+#include "sector.h"
+#include "zilogmcz.h"
+#include "bytes.h"
+#include "crc.h"
+#include "fmt/format.h"
+#include <string.h>
+#include <algorithm>
+
+static const FluxPattern SECTOR_START_PATTERN(16, 0xaaab);
+
+AbstractDecoder::RecordType ZilogMczDecoder::advanceToNextRecord()
+{
+	const FluxMatcher* matcher = nullptr;
+    _fmr->seekToIndexMark();
+	_sector->clock = _fmr->seekToPattern(SECTOR_START_PATTERN, matcher);
+	if (matcher == &SECTOR_START_PATTERN)
+		return SECTOR_RECORD;
+	return UNKNOWN_RECORD;
+}
+
+void ZilogMczDecoder::decodeSectorRecord()
+{
+    readRawBits(14);
+
+    auto rawbits = readRawBits(140*16);
+    auto bytes = decodeFmMfm(rawbits).slice(0, 140);
+    ByteReader br(bytes);
+
+    _sector->logicalSector = br.read_8() & 0x1f;
+    _sector->logicalSide = 0;
+    _sector->logicalTrack = br.read_8() & 0x7f;
+    if (_sector->logicalSector > 31)
+        return;
+    if (_sector->logicalTrack > 80)
+        return;
+
+    _sector->data = br.read(132);
+    uint16_t wantChecksum = br.read_be16();
+    uint16_t gotChecksum = crc16(MODBUS_POLY, 0x0000, bytes.slice(0, 134));
+
+    _sector->status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
+}

arch/zilogmcz/zilogmcz.h

@@ -0,0 +1,18 @@
+#ifndef ZILOGMCZ_H
+#define ZILOGMCZ_H
+
+class Sector;
+class Fluxmap;
+
+class ZilogMczDecoder : public AbstractDecoder
+{
+public:
+    virtual ~ZilogMczDecoder() {}
+
+    RecordType advanceToNextRecord();
+    void decodeSectorRecord();
+};
+
+#endif
+
+

dep/emu/charclass.h

@@ -0,0 +1,29 @@
+/*
+ * Public domain, 2008, Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * $OpenBSD: charclass.h,v 1.1 2008/10/01 23:04:13 millert Exp $
+ */
+
+/*
+ * POSIX character class support for fnmatch() and glob().
+ */
+static struct cclass {
+	const char *name;
+	int (*isctype)(int);
+} cclasses[] = {
+	{ "alnum",	isalnum },
+	{ "alpha",	isalpha },
+	{ "blank",	isblank },
+	{ "cntrl",	iscntrl },
+	{ "digit",	isdigit },
+	{ "graph",	isgraph },
+	{ "lower",	islower },
+	{ "print",	isprint },
+	{ "punct",	ispunct },
+	{ "space",	isspace },
+	{ "upper",	isupper },
+	{ "xdigit",	isxdigit },
+	{ NULL,		NULL }
+};
+
+#define NCCLASSES	(sizeof(cclasses) / sizeof(cclasses[0]) - 1)

dep/emu/fnmatch.c

@@ -0,0 +1,481 @@
+/* 
+ * This has been lightly tweaked by me, David Given, to change the exported name
+ * so that it doesn't conflict with the real fnmatch (if any).
+ */
+
+/*	$OpenBSD: fnmatch.c,v 1.16 2011/12/06 11:47:46 stsp Exp $	*/
+
+/* Copyright (c) 2011, VMware, Inc.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the VMware, Inc. nor the names of its contributors
+ *       may be used to endorse or promote products derived from this software
+ *       without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2008 Todd C. Miller <millert@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/* Authored by William A. Rowe Jr. <wrowe; apache.org, vmware.com>, April 2011
+ *
+ * Derived from The Open Group Base Specifications Issue 7, IEEE Std 1003.1-2008
+ * as described in;
+ *   http://pubs.opengroup.org/onlinepubs/9699919799/functions/fnmatch.html
+ *
+ * Filename pattern matches defined in section 2.13, "Pattern Matching Notation"
+ * from chapter 2. "Shell Command Language"
+ *   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13
+ * where; 1. A bracket expression starting with an unquoted <circumflex> '^' 
+ * character CONTINUES to specify a non-matching list; 2. an explicit <period> '.' 
+ * in a bracket expression matching list, e.g. "[.abc]" does NOT match a leading 
+ * <period> in a filename; 3. a <left-square-bracket> '[' which does not introduce
+ * a valid bracket expression is treated as an ordinary character; 4. a differing
+ * number of consecutive slashes within pattern and string will NOT match;
+ * 5. a trailing '\' in FNM_ESCAPE mode is treated as an ordinary '\' character.
+ *
+ * Bracket expansion defined in section 9.3.5, "RE Bracket Expression",
+ * from chapter 9, "Regular Expressions"
+ *   http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05
+ * with no support for collating symbols, equivalence class expressions or 
+ * character class expressions.  A partial range expression with a leading 
+ * hyphen following a valid range expression will match only the ordinary
+ * <hyphen> and the ending character (e.g. "[a-m-z]" will match characters 
+ * 'a' through 'm', a <hyphen> '-', or a 'z').
+ *
+ * Supports BSD extensions FNM_LEADING_DIR to match pattern to the end of one
+ * path segment of string, and FNM_CASEFOLD to ignore alpha case.
+ *
+ * NOTE: Only POSIX/C single byte locales are correctly supported at this time.
+ * Notably, non-POSIX locales with FNM_CASEFOLD produce undefined results,
+ * particularly in ranges of mixed case (e.g. "[A-z]") or spanning alpha and
+ * nonalpha characters within a range.
+ *
+ * XXX comments below indicate porting required for multi-byte character sets
+ * and non-POSIX locale collation orders; requires mbr* APIs to track shift
+ * state of pattern and string (rewinding pattern and string repeatedly).
+ *
+ * Certain parts of the code assume 0x00-0x3F are unique with any MBCS (e.g.
+ * UTF-8, SHIFT-JIS, etc).  Any implementation allowing '\' as an alternate
+ * path delimiter must be aware that 0x5C is NOT unique within SHIFT-JIS.
+ */
+
+#include "fnmatch.h"
+#include <string.h>
+#include <ctype.h>
+#include <limits.h>
+
+#include "charclass.h"
+
+#define	RANGE_MATCH	1
+#define	RANGE_NOMATCH	0
+#define	RANGE_ERROR	(-1)
+
+static int
+classmatch(const char *pattern, char test, int foldcase, const char **ep)
+{
+	struct cclass *cc;
+	const char *colon;
+	size_t len;
+	int rval = RANGE_NOMATCH;
+	const char * const mismatch = pattern;
+
+	if (*pattern != '[' || pattern[1] != ':') {
+		*ep = mismatch;
+		return(RANGE_ERROR);
+	}
+
+	pattern += 2;
+
+	if ((colon = strchr(pattern, ':')) == NULL || colon[1] != ']') {
+		*ep = mismatch;
+		return(RANGE_ERROR);
+	}
+	*ep = colon + 2;
+	len = (size_t)(colon - pattern);
+
+	if (foldcase && strncmp(pattern, "upper:]", 7) == 0)
+		pattern = "lower:]";
+	for (cc = cclasses; cc->name != NULL; cc++) {
+		if (!strncmp(pattern, cc->name, len) && cc->name[len] == '\0') {
+			if (cc->isctype((unsigned char)test))
+				rval = RANGE_MATCH;
+			break;
+		}
+	}
+	if (cc->name == NULL) {
+		/* invalid character class, treat as normal text */
+		*ep = mismatch;
+		rval = RANGE_ERROR;
+	}
+	return(rval);
+}
+
+/* Most MBCS/collation/case issues handled here.  Wildcard '*' is not handled.
+ * EOS '\0' and the FNM_PATHNAME '/' delimiters are not advanced over, 
+ * however the "\/" sequence is advanced to '/'.
+ *
+ * Both pattern and string are **char to support pointer increment of arbitrary
+ * multibyte characters for the given locale, in a later iteration of this code
+ */
+static int fnmatch_ch(const char **pattern, const char **string, int flags)
+{
+    const char * const mismatch = *pattern;
+    const int nocase = !!(flags & FNM_CASEFOLD);
+    const int escape = !(flags & FNM_NOESCAPE);
+    const int slash = !!(flags & FNM_PATHNAME);
+    int result = FNM_NOMATCH;
+    const char *startch;
+    int negate;
+
+    if (**pattern == '[')
+    {
+        ++*pattern;
+
+        /* Handle negation, either leading ! or ^ operators (never both) */
+        negate = ((**pattern == '!') || (**pattern == '^'));
+        if (negate)
+            ++*pattern;
+
+        /* ']' is an ordinary character at the start of the range pattern */
+        if (**pattern == ']')
+            goto leadingclosebrace;
+
+        while (**pattern)
+        {
+            if (**pattern == ']') {
+                ++*pattern;
+                /* XXX: Fix for MBCS character width */
+                ++*string;
+                return (result ^ negate);
+            }
+
+            if (escape && (**pattern == '\\')) {
+                ++*pattern;
+
+                /* Patterns must be terminated with ']', not EOS */
+                if (!**pattern)
+                    break;
+            }
+
+            /* Patterns must be terminated with ']' not '/' */
+            if (slash && (**pattern == '/'))
+                break;
+
+            /* Match character classes. */
+            if (classmatch(*pattern, **string, nocase, pattern)
+                == RANGE_MATCH) {
+                result = 0;
+                continue;
+            }
+
+leadingclosebrace:
+            /* Look at only well-formed range patterns; 
+             * "x-]" is not allowed unless escaped ("x-\]")
+             * XXX: Fix for locale/MBCS character width
+             */
+            if (((*pattern)[1] == '-') && ((*pattern)[2] != ']'))
+            {
+                startch = *pattern;
+                *pattern += (escape && ((*pattern)[2] == '\\')) ? 3 : 2;
+
+                /* NOT a properly balanced [expr] pattern, EOS terminated 
+                 * or ranges containing a slash in FNM_PATHNAME mode pattern
+                 * fall out to to the rewind and test '[' literal code path
+                 */
+                if (!**pattern || (slash && (**pattern == '/')))
+                    break;
+
+                /* XXX: handle locale/MBCS comparison, advance by MBCS char width */
+                if ((**string >= *startch) && (**string <= **pattern))
+                    result = 0;
+                else if (nocase && (isupper(**string) || isupper(*startch)
+                                                      || isupper(**pattern))
+                            && (tolower(**string) >= tolower(*startch)) 
+                            && (tolower(**string) <= tolower(**pattern)))
+                    result = 0;
+
+                ++*pattern;
+                continue;
+            }
+
+            /* XXX: handle locale/MBCS comparison, advance by MBCS char width */
+            if ((**string == **pattern))
+                result = 0;
+            else if (nocase && (isupper(**string) || isupper(**pattern))
+                            && (tolower(**string) == tolower(**pattern)))
+                result = 0;
+
+            ++*pattern;
+        }
+
+        /* NOT a properly balanced [expr] pattern; Rewind
+         * and reset result to test '[' literal
+         */
+        *pattern = mismatch;
+        result = FNM_NOMATCH;
+    }
+    else if (**pattern == '?') {
+        /* Optimize '?' match before unescaping **pattern */
+        if (!**string || (slash && (**string == '/')))
+            return FNM_NOMATCH;
+        result = 0;
+        goto fnmatch_ch_success;
+    }
+    else if (escape && (**pattern == '\\') && (*pattern)[1]) {
+        ++*pattern;
+    }
+
+    /* XXX: handle locale/MBCS comparison, advance by the MBCS char width */
+    if (**string == **pattern)
+        result = 0;
+    else if (nocase && (isupper(**string) || isupper(**pattern))
+                    && (tolower(**string) == tolower(**pattern)))
+        result = 0;
+
+    /* Refuse to advance over trailing slash or nulls
+     */
+    if (!**string || !**pattern || (slash && ((**string == '/') || (**pattern == '/'))))
+        return result;
+
+fnmatch_ch_success:
+    ++*pattern;
+    ++*string;
+    return result;
+}
+
+
+int fnmatch(const char *pattern, const char *string, int flags)
+{
+    static const char dummystring[2] = {' ', 0};
+    const int escape = !(flags & FNM_NOESCAPE);
+    const int slash = !!(flags & FNM_PATHNAME);
+    const char *strendseg;
+    const char *dummyptr;
+    const char *matchptr;
+    int wild;
+    /* For '*' wild processing only; surpress 'used before initialization'
+     * warnings with dummy initialization values;
+     */
+    const char *strstartseg = NULL;
+    const char *mismatch = NULL;
+    int matchlen = 0;
+
+    if (strnlen(pattern, PATH_MAX) == PATH_MAX ||
+        strnlen(string, PATH_MAX) == PATH_MAX)
+            return (FNM_NOMATCH);
+
+    if (*pattern == '*')
+        goto firstsegment;
+
+    while (*pattern && *string)
+    {
+        /* Pre-decode "\/" which has no special significance, and
+         * match balanced slashes, starting a new segment pattern
+         */
+        if (slash && escape && (*pattern == '\\') && (pattern[1] == '/'))
+            ++pattern;
+        if (slash && (*pattern == '/') && (*string == '/')) {
+            ++pattern;
+            ++string;
+        }            
+
+firstsegment:
+        /* At the beginning of each segment, validate leading period behavior.
+         */
+        if ((flags & FNM_PERIOD) && (*string == '.'))
+        {
+            if (*pattern == '.')
+                ++pattern;
+            else if (escape && (*pattern == '\\') && (pattern[1] == '.'))
+                pattern += 2;
+            else
+                return FNM_NOMATCH;
+            ++string;
+        }
+
+        /* Determine the end of string segment
+         *
+         * Presumes '/' character is unique, not composite in any MBCS encoding
+         */
+        if (slash) {
+            strendseg = strchr(string, '/');
+            if (!strendseg)
+                strendseg = strchr(string, '\0');
+        }
+        else {
+            strendseg = strchr(string, '\0');
+        }
+
+        /* Allow pattern '*' to be consumed even with no remaining string to match
+         */
+        while (*pattern)
+        {
+            if ((string > strendseg)
+                || ((string == strendseg) && (*pattern != '*')))
+                break;
+
+            if (slash && ((*pattern == '/')
+                           || (escape && (*pattern == '\\')
+                                      && (pattern[1] == '/'))))
+                break;
+
+            /* Reduce groups of '*' and '?' to n '?' matches
+             * followed by one '*' test for simplicity
+             */
+            for (wild = 0; ((*pattern == '*') || (*pattern == '?')); ++pattern)
+            {
+                if (*pattern == '*') {
+                    wild = 1;
+                }
+                else if (string < strendseg) {  /* && (*pattern == '?') */
+                    /* XXX: Advance 1 char for MBCS locale */
+                    ++string;
+                }
+                else {  /* (string >= strendseg) && (*pattern == '?') */
+                    return FNM_NOMATCH;
+                }
+            }
+
+            if (wild)
+            {
+                strstartseg = string;
+                mismatch = pattern;
+
+                /* Count fixed (non '*') char matches remaining in pattern
+                 * excluding '/' (or "\/") and '*'
+                 */
+                for (matchptr = pattern, matchlen = 0; 1; ++matchlen)
+                {
+                    if ((*matchptr == '\0') 
+                        || (slash && ((*matchptr == '/')
+                                      || (escape && (*matchptr == '\\')
+                                                 && (matchptr[1] == '/')))))
+                    {
+                        /* Compare precisely this many trailing string chars,
+                         * the resulting match needs no wildcard loop
+                         */
+                        /* XXX: Adjust for MBCS */
+                        if (string + matchlen > strendseg)
+                            return FNM_NOMATCH;
+
+                        string = strendseg - matchlen;
+                        wild = 0;
+                        break;
+                    }
+
+                    if (*matchptr == '*')
+                    {
+                        /* Ensure at least this many trailing string chars remain
+                         * for the first comparison
+                         */
+                        /* XXX: Adjust for MBCS */
+                        if (string + matchlen > strendseg)
+                            return FNM_NOMATCH;
+
+                        /* Begin first wild comparison at the current position */
+                        break;
+                    }
+
+                    /* Skip forward in pattern by a single character match
+                     * Use a dummy fnmatch_ch() test to count one "[range]" escape
+                     */ 
+                    /* XXX: Adjust for MBCS */
+                    if (escape && (*matchptr == '\\') && matchptr[1]) {
+                        matchptr += 2;
+                    }
+                    else if (*matchptr == '[') {
+                        dummyptr = dummystring;
+                        fnmatch_ch(&matchptr, &dummyptr, flags);
+                    }
+                    else {
+                        ++matchptr;
+                    }
+                }
+            }
+
+            /* Incrementally match string against the pattern
+             */
+            while (*pattern && (string < strendseg))
+            {
+                /* Success; begin a new wild pattern search
+                 */
+                if (*pattern == '*')
+                    break;
+
+                if (slash && ((*string == '/')
+                              || (*pattern == '/')
+                              || (escape && (*pattern == '\\')
+                                         && (pattern[1] == '/'))))
+                    break;
+
+                /* Compare ch's (the pattern is advanced over "\/" to the '/',
+                 * but slashes will mismatch, and are not consumed)
+                 */
+                if (!fnmatch_ch(&pattern, &string, flags))
+                    continue;
+
+                /* Failed to match, loop against next char offset of string segment 
+                 * until not enough string chars remain to match the fixed pattern
+                 */
+                if (wild) {
+                    /* XXX: Advance 1 char for MBCS locale */
+                    string = ++strstartseg;
+                    if (string + matchlen > strendseg)
+                        return FNM_NOMATCH;
+
+                    pattern = mismatch;
+                    continue;
+                }
+                else
+                    return FNM_NOMATCH;
+            }
+        }
+
+        if (*string && !(slash && (*string == '/')))
+            return FNM_NOMATCH;
+
+        if (*pattern && !(slash && ((*pattern == '/')
+                                    || (escape && (*pattern == '\\')
+                                               && (pattern[1] == '/')))))
+            return FNM_NOMATCH;
+    }
+
+    /* Where both pattern and string are at EOS, declare success
+     */
+    if (!*string && !*pattern)
+        return 0;
+
+    /* pattern didn't match to the end of string */
+    return FNM_NOMATCH;
+}

dep/emu/fnmatch.h

@@ -0,0 +1,21 @@
+#ifndef FNMATCH_H
+#define FNMATCH_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	FNM_NOMATCH	1	/* Match failed. */
+
+#define	FNM_NOESCAPE 0x01 /* Disable backslash escaping. */
+#define	FNM_PATHNAME 0x02 /* Slash must be matched by slash. */
+#define	FNM_PERIOD   0x04 /* Period must be matched by period. */
+#define FNM_CASEFOLD 0x08 /* Fold cases */
+
+extern int fnmatch(const char *pattern, const char *string, int flags);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

dep/fmt/fmt/format.h

@@ -3416,7 +3416,7 @@ auto join(const Range &range, wstring_view sep)
 
   **Example**::
 
-    #include <fmt/format.h>
+    #include "fmt/format.h"
 
     std::string answer = fmt::to_string(42);
   \endrst
@@ -3697,7 +3697,7 @@ FMT_END_NAMESPACE
   **Example**::
 
     #define FMT_STRING_ALIAS 1
-    #include <fmt/format.h>
+    #include "fmt/format.h"
     // A compile-time error because 'd' is an invalid specifier for strings.
     std::string s = format(fmt("{:d}"), "foo");
   \endrst

doc/building.md

@@ -103,7 +103,43 @@ the unconnected pins and solder a short piece of wire to a GND pin on the
 board. Alternatively you'll need to splice it into your drive's power supply
 cable somehow. (The black one.)
 
-## Building the firmware
+## Programming the board
+
+You've got two options here. You can either use the precompiled firmware
+supplied with the source, or else install the Cypress SDK and build it
+yourself. If you want to hack the firmware source you need the latter, but
+if you trust me to do it for you use the precompiled firmware. In either
+case you'll need Windows and have to install some Cypress stuff.
+
+**Before you read this:** If you're on Windows, good news! You can download a
+precompiled version of the FluxEngine client and precompiled firmware [from
+the GitHub releases
+page](https://github.com/davidgiven/fluxengine/releases/latest). Simply unzip
+it somewhere and run the `.exe` files from a `cmd` window (or other shell).
+Follow the instructions below to program the board with the firmware.
+
+### Using the precompiled firmware
+
+On your Windows machine, [install the PSoC
+Programmer](https://www.cypress.com/products/psoc-programming-solutions).
+**Note:** _not_ the Cypress Programmer, which is for a different board!
+Cypress will make you register.
+
+Once done, run it. Plug the blunt end of the FluxEngine board into a USB
+port (the end which is a USB connector). The programmer should detect it
+and report it as a KitProg. You may be prompted to upgrade the programmer
+hardware; if so, follow the instructions and do it.
+
+Now go to File -> File Load and open
+`FluxEngine.cydsn/CortexM3/ARM_GCC_541/Release/FluxEngine.hex` in the
+project. If you're on Windows, the precompiled zipfile also contains a copy
+of this file. Press the Program button (the one in the toolbar marked with a
+down arrow). Stuff will happen and you should be left with three green boxes
+in the status bar and 'Programming Succeeded' at the top of the log window.
+
+You're done. You can unplug the board and close the programmer.
+
+### Building the firmware yourself
 
 On your Windows machine, [install the Cypress SDK and CY8CKIT-059
 BSP](http://www.cypress.com/documentation/development-kitsboards/cy8ckit-059-psoc-5lp-prototyping-kit-onboard-programmer-and).
@@ -118,7 +154,7 @@ tutorial and making the LED on your board flash. It'll tell you where all the
 controls are and how to program the board. Remember that the big end of the
 board plugs into your computer for programming.
 
-When you're ready, open the `FluxEngine.cydsn/FluxEngine.cywrk` workspace,
+When you're ready, open the `FluxEngine.cydsn/FluxEngine.cyprj` project,
 pick 'Program' from the menu, and the firmware should compile and be
 programmed onto your board.
 
@@ -141,16 +177,31 @@ the port and proceed normally.
 
 The client software is where the intelligence, such as it is, is. It's pretty
 generic libusb stuff and should build and run on Windows, Linux and OSX as
-well, although on Windows I've only ever used it with Cygwin. You'll need the
-`sqlite3`, `meson` and `ninja` packages (which should be easy to come by in
-your distribution). Just do `make` and it should build.
+well, although on Windows it'll need MSYS2 and mingw32. You'll need to
+install some support packages.
+
+  - For Linux (this is Ubuntu, but this should apply to Debian too):
+  `ninja-build`, `libusb-1.0-0-dev`, `libsqlite3-dev`.
+  - For OSX with Homebrew: `ninja`.
+  - For Windows with MSYS2: `make`, `ninja`, `mingw-w64-i686-libusb`,
+  `mingw-w64-i686-sqlite3`, `mingw-w64-i686-zlib`, `mingw-w64-i686-gcc`.
+
+These lists are not necessarily exhaustive --- plaese [get in
+touch](https://github.com/davidgiven/fluxengine/issues/new) if I've missed
+anything.
+
+All systems build by just doing `make`. You should end up with a single
+executable in the current directory, called `fluxengine`. It has minimal
+dependencies and you should be able to put it anywhere.
 
 If it doesn't build, please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new).
 
 ## Next steps
 
-The board's now assembled and programmed. Plug it into your drive, strip the plastic off the little USB connector and plug that into your computer, and you're ready to start using it.
+The board's now assembled and programmed. Plug it into your drive, strip the
+plastic off the little USB connector and plug that into your computer, and
+you're ready to start using it.
 
 I _do_ make updates to the firmware whenever necessary, so you may need to
 reprogram it at intervals; you may want to take this into account if you

doc/disk-acornadfs.md

@@ -31,7 +31,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readadfs
+fluxengine read adfs
 ```
 
 You should end up with an `adfs.img` of the appropriate size for your disk

doc/disk-acorndfs.md

@@ -19,7 +19,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readdfs
+fluxengine read dfs
 ```
 
 You should end up with an `dfs.img` of the appropriate size for your disk

doc/disk-aeslanier.md

@@ -3,7 +3,7 @@ Disk: AES Lanier word processor
 
 Back in 1980 Lanier released a series of very early integrated word processor
 appliances, the No Problem. These were actually [rebranded AES Data Superplus
-machines](http://vintagecomputers.site90.net/aes/). They wrer gigantic,
+machines](http://vintagecomputers.site90.net/aes/). They were gigantic,
 weighed 40kg, and one example I've found cost £13,000 in 1981 (the equivalent
 of nearly £50,000 in 2018!).
 
@@ -17,9 +17,10 @@ indicating to the hardware where the sectors start. The encoding scheme
 itself is [MMFM (aka
 M2FM)](http://www.retrotechnology.com/herbs_stuff/m2fm.html), an early
 attempt at double-density disk encoding which rapidly got obsoleted by the
-simpler MFM. Even aside from the encoding, the format on disk was strange;
-unified sector header/data records, so that the sector header (containing the
-sector and track number) is actually inside the user data.
+simpler MFM --- and the bytes are stored on disk _backwards_. Even aside from
+the encoding, the format on disk was strange; unified sector header/data
+records, so that the sector header (containing the sector and track number)
+is actually inside the user data.
 
 FluxEngine can read these, but I only have a single, fairly poor example of a
 disk image, and I've had to make a lot of guesses as to the sector format
@@ -32,10 +33,10 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readaeslanier
+fluxengine read aeslanier
 ```
 
 Useful references
 -----------------
 
-  * [SA800 Diskette Storage Drive - Theory Of Operations](http://www.hartetechnologies.com/manuals/Shugart/50664-1_SA800_TheorOp_May78.pdf): talks about MMFM a lot, but the Lanier machines didn't use this disk format.
+  * [SA800 Diskette Storage Drive - Theory Of Operations](http://www.hartetechnologies.com/manuals/Shugart/50664-1_SA800_TheorOp_May78.pdf): talks about MMFM a lot, but the Lanier machines didn't use this disk format.

doc/disk-amiga.md

@@ -16,13 +16,24 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readamiga
+fluxengine read amiga
 ```
 
 You should end up with an `amiga.adf` which is 901120 bytes long (for a
 normal DD disk) --- it ought to be a perfectly normal ADF file which you can
 use in an emulator.
 
+If you want the metadata as well, specify a 528 byte sector size for the
+output image:
+
+```
+fluxengine read amiga -o amiga.adf:b=528
+```
+
+You will end up with a 929280 byte long image which you probably _can't_ use
+in an emulator; each sector will contain the 512 bytes of user payload
+followed by the 16 bytes of metadata.
+
 Useful references
 -----------------
 

doc/disk-ampro.md

@@ -19,7 +19,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readampro
+fluxengine read ampro
 ```
 
 You should end up with an `ampro.img` which is 409600 or 819200 bytes long.

doc/disk-apple2.md

@@ -17,9 +17,9 @@ scheme applied to the data before it goes down on disk to speed up
 checksumming.
 
 Macintosh disks come in two varieties: the newer 1440kB ones, which are
-perfectly ordinary PC disks you should use `fe-readibm` to read, and the
-older 800kB disks (and 400kB for the single sides ones). They have 80 tracks
-and up to 12 sectors per track.
+perfectly ordinary PC disks you should use `fluxengine read ibm` to read, and
+the older 800kB disks (and 400kB for the single sides ones). They have 80
+tracks and up to 12 sectors per track.
 
 In addition, a lot of the behaviour of the drive was handled in software.
 This means that Apple II disks can do all kinds of weird things, including
@@ -42,7 +42,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readapple2
+fluxengine read apple2
 ```
 
 You should end up with an `apple2.img` which is 143360 bytes long.

doc/disk-brother.md

@@ -8,7 +8,7 @@ size.
 Different word processors use different disk formats --- the only ones
 supported by FluxEngine are the 120kB and 240kB 3.5" formats. The default
 options are for the 240kB format. For the 120kB format, which is 40 track, do
-`fe-readbrother -s :t=1-79x2`.
+`fluxengine read brother -s :t=1-79x2`.
 
 Apparently about 20% of Brother word processors have alignment issues which
 means that the disks can't be read by FluxEngine (because the tracks on the
@@ -36,7 +36,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readbrother
+fluxengine read brother
 ```
 
 You should end up with a `brother.img` which is 239616 bytes long.
@@ -47,7 +47,7 @@ Writing discs
 Just do:
 
 ```
-.obj/fe-writebrother
+fluxengine write brother
 ```
 
 ...and it'll write a `brother.img` file which is 239616 bytes long to the
@@ -108,9 +108,13 @@ To extract a file, do:
 Wildcards are supported, so use `'*'` for the filename (remember to quote it)
 if you want to extract everything.
 
-This is _extremely experimental_. The data structures I've figured out are
-mostly consistent, but it looks like there's always garbage in the last
-sector of each file, so maybe I'm not getting the file lengths right.
+The files are usually in the format known as WP-1, which aren't well
+supported by modern tools (to nobody's great surprise). Matthias Henckell has
+[reverse engineered the file
+format](https://mathesoft.eu/brother-wp-1-dokumente/) and has produced a
+(Windows-only, but runs in Wine) [tool which will convert these files into
+RTF](https://mathesoft.eu/sdm_downloads/wp2rtf/). This will only work on WP-1
+files.
 
 Any questions? please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new).
@@ -136,5 +140,6 @@ mcopy -i brother.img ::brother.doc linux.doc
 The word processor checks the media byte, unfortunately, so you'll need to
 change it back to 0x58 before writing an image to disk.
 
-Converting the equally proprietary file format to something readable is,
-unfortunately, out of scope for FluxEngine.
+The file format is not WP-1, and currently remains completely unknown,
+although it's probably related. If anyone knows anything about this, please
+[get in touch](https://github.com/davidgiven/fluxengine/issues/new).

doc/disk-c64.md

@@ -3,8 +3,8 @@ Disk: Commodore 64
 
 Commodore 64 disks come in two varieties: GCR, which are the overwhelming
 majority; and MFM, only used on the 1571 and 1581. The latter were (as far as
-I can tell) standard IBM PC format disks, so use `fe-readibm` to read them
-(and then [let me know if it
+I can tell) standard IBM PC format disks, so use `fluxengine read ibm` to
+read them (and then [let me know if it
 worked](https://github.com/davidgiven/fluxengine/issues/new).
 
 The GCR disks are much more interesting. They could store 170kB on a
@@ -23,7 +23,7 @@ computer](https://ilesj.wordpress.com/2014/05/14/1541-why-so-complicated/) of
 300 bytes per second (!). (The drive itself could transfer data reasonably
 quickly.)
 
-A standard 1541 disk has 35 tracks of 17 to 20 sectors, each 256 bytes long.
+A standard 1541 disk has 35 tracks of 17 to 21 sectors, each 256 bytes long.
 
 Reading discs
 -------------
@@ -31,18 +31,17 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readc64
+fluxengine read c64
 ```
 
-You should end up with an `c64.img` which is 187136 bytes long (for a normal
-1541 disk).
+You should end up with an `c64.d64` file which is 174848 bytes long. You can
+load this straight into a Commodore 64 emulator such as
+[VICE](http://vice-emu.sourceforge.net/).
 
-**Big warning!** The image may not work in an emulator. Commodore 64 disk images are
+**Big warning!** Commodore 64 disk images are
 complicated due to the way the tracks are different sizes and the odd sector
-size. FluxEngine chooses to store them in a simple 256 x 20 x 35 layout,
-with holes where missing sectors should be. This was easiest. If anyone can
-suggest a better way, please [get in
-touch](https://github.com/davidgiven/fluxengine/issues/new).
+size, so you need the special D64 or LDBS output formats to represent them
+sensibly. Don't use IMG unless you know what you're doing.
 
 Useful references
 -----------------

doc/disk-durangof85.md

@@ -15,7 +15,8 @@ it for the following photo...
 <img src="durangof85.jpg" style="max-width: 60%" alt="A Durango F85, held precariously">
 </div>
 
-...and even then, only for a few seconds.
+...and even then, they had to airbrush out the tendons in her neck from the
+effort!
 
 It used 5.25 soft-sectored disks storing an impressive-for-those-days
 480kBish on a side, using a proprietary 4-in-5 GCR encoding. They used 77
@@ -31,7 +32,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readf85
+fluxengine read f85
 ```
 
 You should end up with an `f85.img` which is 472064 bytes long.

doc/disk-fb100.md

@@ -0,0 +1,45 @@
+Disk: Brother FB-100
+====================
+
+The Brother FB-100 is a serial-attached smart floppy drive used by a several
+different machines for mass storage, including the Tandy Model 100 and
+clones, the Husky Hunter 2, and (bizarrely) several knitting machines. It was
+usually rebadged, sometimes with a cheap paper label stuck over the Brother
+logo, but the most common variant appears to be the Tandy Portable Disk Drive
+or TPDD:
+
+<div style="text-align: center">
+<a href="http://www.old-computers.com/museum/computer.asp?c=233&st=1"> <img src="tpdd.jpg" alt="A Tandy Portable Disk Drive"/></a>
+</div>
+
+It's a bit of an oddball: the disk encoding is FM with a very custom record
+scheme: 40-track single-sided 3.5" disks storing 100kB or so each. Each track
+had only _two_ sectors, each 1280 bytes, but with an additional 12 bytes of
+ID data used for filesystem management.
+
+There was also apparently a TPDD-2 which could store twice as much data, but
+I don't have access to one of those disks.
+
+Reading discs
+-------------
+
+Just do:
+
+```
+fluxengine read fb100
+```
+
+You should end up with an `fb100.img` of the appropriate size. It's a simple
+array of 80 1292-byte sectors (12 bytes for the ID record plus 1280 bytes for
+the data).
+
+References
+----------
+
+  - [Tandy Portable Disk Drive operations manual](http://www.classiccmp.org/cini/pdf/Tandy/Portable%20Disk%20Drive%20Operation%20Manual.pdf)
+
+  - [Tandy Portable Disk Drive service manual](https://archive.org/details/TandyPortableDiskDriveSoftwareManual26-3808s)
+
+  - [TPDD design notes (including a dump of the ROM)](http://bitchin100.com/wiki/index.php?title=TPDD_Design_Notes)
+
+  - [Knitting machine FB-100 resources](http://www.k2g2.org/wiki:brother_fb-100)

doc/disk-macintosh.md

@@ -2,9 +2,9 @@ Disk: Macintosh
 ===============
 
 Macintosh disks come in two varieties: the newer 1440kB ones, which are
-perfectly ordinary PC disks you should use `fe-readibm` to read, and the
-older 800kB disks (and 400kB for the single sides ones). They have 80 tracks
-and up to 12 sectors per track.
+perfectly ordinary PC disks you should use `fluxengine read ibm` to read, and
+the older 800kB disks (and 400kB for the single sides ones). They have 80
+tracks and up to 12 sectors per track.
 
 They are also completely insane.
 
@@ -37,7 +37,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readmac
+fluxengine read mac
 ```
 
 You should end up with an `mac.img` which is 1001888 bytes long (for a normal
@@ -50,6 +50,10 @@ with holes where missing sectors should be. This was easiest. If anyone can
 suggest a better way, please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new).
 
+The 12 bytes of metadata _follow_ the 512 bytes of user payload in the sector
+image. If you don't want it, specify a geometry in the output file with a
+512-byte sectore size like `-o mac.img:c=80:h=1:s=12:b=512`.
+
 Useful references
 -----------------
 

doc/disk-mx.md

@@ -0,0 +1,65 @@
+Disk: DVK MX
+============
+
+The DVK (in Russian, ДВК, Диалоговый вычислительный комплекс or Dialogue
+Computing Complex) was a late 1970s Soviet personal computer, a cut-down
+version of the professional SM EVM (СМ ЭВМ, abbreviation of Система Малых ЭВМ
+--- literally System of Mini Computers), which _itself_ was an unlicensed
+clone of the PDP-11. The MX board was an early floppy drive controller board
+for it.
+
+<div style="text-align: center">
+<a href="http://www.leningrad.su/museum/show_big.php?n=1006"><img src="dvk3m.jpg" style="max-width: 60%" alt="A Durango F85, held precariously"></a>
+</div>
+
+The MX format is interesting in that it has to be read a track at a time. The
+format contains the usual ID prologue at the beginning of the track, then
+eleven data blocks and checksums, then the epilogue, then it stops. The
+actual encoding is normal FM. There were four different disk variants, in all
+combinations of single- and double-sided and 40- and 80-tracked; but every
+track contained eleven 256-byte sectors.
+
+The format varies subtly depending on whether you're using the 'new' driver
+or the 'old' driver. FluxEngine should read both.
+
+A track is:
+
+  * 8 x 0x0000 words (FM encoded as 01010101...)
+  * 1 x 0x00F3 --- start of track
+  * 1 x 0xnnnn --- track number
+  * 11 of:
+    * 128 words (256 bytes) of data
+    * 16 bit checksum
+  * **if 'new' format:**
+    * 3 x 0x83nn --- `n = (track_number<<1) + side_number`
+  * **if 'old' format:**
+    * 3 x 0x8301
+
+The checksum is just the unsigned integer sum of all the words in the sector.
+Words are all stored little-endian.
+
+Reading discs
+-------------
+
+```
+fluxengine read mx
+```
+
+You should end up with an `mx.img` which will vary in length depending on the format. The default is double-sided 80-track. For the other formats, use:
+
+  * single-sided 40-track: `-s :s=0:t=0-79x2`
+  * double-sided 40-track: `-s :s=0-1:t=0-79x2`
+  * single-sided 40-track: `-s :s=0:t=0-79`
+  * double-sided 40-track: `-s :s=0-1:t=0-79`
+
+
+Useful references
+-----------------
+
+  - [The Soviet Digital Electronics
+    Museum](http://www.leningrad.su/museum/main.php) (source of the image
+    above)
+
+  - [a random post on the HxC2001 support
+    forum](http://torlus.com/floppy/forum/viewtopic.php?t=1384) with lots of
+    information on the format

doc/disk-trs80.md

@@ -22,7 +22,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readibm
+fluxengine read ibm
 ```
 
 You should end up with an `ibm.img` of the appropriate size. It's a simple

doc/disk-victor9k.md

@@ -1,22 +1,14 @@
 Disk: Victor 9000
 =================
 
-**Warning.** This is experimental; I haven't found a clean disk to read yet.
-The fragmented disk images which I have found ([from
-vintagecomputer.ca](http://vintagecomputer.ca/files/Victor%209000/)) get
-about 57% good sectors. It could just be that the disks are bad, but there
-could also be something wrong with my decode logic. If you have any Victor
-disks and want to give this a try for real, [please get in
-touch](https://github.com/davidgiven/fluxengine/issues/new).
-
 The Victor 9000 / Sirius One was a rather strange old 8086-based machine
 which used a disk format very reminiscent of the Commodore format; not a
 coincidence, as Chuck Peddle designed them both. They're 80-track, 512-byte
 sector GCR disks, with a variable-speed drive and a varying number of sectors
-per track --- from 19 to 12. Reports are that they're double-sided but I've
-never seen a double-sided disk image.
+per track --- from 19 to 12. Disks can be double-sided, meaning that they can
+store 1224kB per disk, which was almost unheard of back then.
 
-FluxEngine reads them (subject to the warning above).
+FluxEngine reads these.
 
 Reading discs
 -------------
@@ -24,7 +16,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readvictor9k
+fluxengine read victor9k
 ```
 
 You should end up with an `victor9k.img` which is 774656 bytes long.

doc/disk-zilogmcz.md

@@ -30,7 +30,7 @@ Reading discs
 Just do:
 
 ```
-.obj/fe-readzilogmcz
+fluxengine read zilogmcz
 ```
 
 You should end up with an `zilogmcz.img` which is 315392 bytes long.

doc/problems.md

@@ -1,64 +1,146 @@
 My disk won't read properly!
 ============================
 
-So you're trying to read a disk and it's not working. Well, you're not the only one. Floppy disks are unreliable and awkward things. FluxEngine can help, but the tools aren't very user friendly.
+So you're trying to read a disk and it's not working. Well, you're not the
+only one. Floppy disks are unreliable and awkward things. FluxEngine can
+help, but the tools aren't very user friendly.
+
+The good news is that as of 2019-04-30 the FluxEngine decoder algorithm has
+been written to be largely fire-and-forget and is mostly self-adjusting.
+However, there are still some things that can be tuned to produce better
+reads.
 
 The sector map
 --------------
 
-Every time I do a read, FluxEngine will give me a dump like this:
+Every time FluxEngine does a read, it produces a dump like this:
 
 ```
 H.SS Tracks --->
-0. 0 XBBXXXXXXXXXBXBBXBBXBBX..XXX.BXBBBXXX....BB...BB...........B.XXXX.X....BBBBBBXBB
-0. 1 X.BXXXXBBXXX.XBBXBXXBBB.XXXX.BBXBXXBX....BB...BX........BB.B.XXXX.X....BBBBBBXBX
-0. 2 X..XXXXXBXXX.XBBXXBXBBB.BXXB.BXBBBXXX...BXX.B.BB.........B.B.XXXX.X....BBBBBBXBX
-0. 3 X.BXXXXXXXXX.X.XXBBXBBX.XXXX.XXXXXXXX...BBX.X.XX......B....B.XXXB.X....BBBXBBXBX
-0. 4 X.BXXXXXXXXX.XBXXBXXBBB.XXXX.BXXXXXXX...BBB.X.BX......B....B.XXXX.X....BBBBBBXBX
-0. 5 X.BXXXXXXXXX.XBXXBBXBB.BXXXX.XXXXXXXX...XBB.X.XX.....B.....B.XXXX.B....BBBXBBXBX
-0. 6 X..XXXXBXXXX.X.XXBBXBBB.XXXB.XXBBBXXX..B.BB.B.XB.............XXXX.X....BBBXBBXBX
-0. 7 X..XXXXBXXXXBX.XXBBXBBB.BXXX.XXXBXXXX..BBXX.X.XX......B....B.XXXX.X....BBBXBBXBX
-0. 8 X.BXXXXBXXXXBX.XXBBXBBX.XXXX.XXXXXXXX..BXXX.X.XX......BB.B...XXXX.X....BBBBXBXBX
-0. 9 X.BXXXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX..XXXX.X.XX......X......XXXX.X....BBBBBBXBX
-0.10 X..XXXXBXXXXBX.XXBBXXB..BXXXBBXXBXXXX..BBXB.B.BB.............XXXX.X....BBBBBXXBX
-0.11 X..XXXXXXXXXBX.XXBBXBBB.XXXX.BXXXXXXX..XXXX.X.BB....B........XXXX.X....BBBBBBBBB
-0.12 X.BXXXXXXXXXBX.XXBXXXXB.XXXX.XXXXXXXX..XXXX.X.XX.........B.B.BXXX.B....XXXXXXXXX
-0.13 X..XBXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX..XXXX.X.XX....B.B.BB.BXXXXXXXXXXXXXXXXXXXX
-0.14 XB.XXXXXXXXXBX.XXBBXBBB.XXXB.XXBXXXXB..BBB..B.BBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.15 X..XXXXXXXXXBX.XXBBXBBB.XXXX.XXXXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.16 X..XXBXXBXBXBBBXXBBXBBB.XXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.17 XBBXXXXB.XXXBXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.18 XBBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 369/1520 (24%)
-Missing sectors: 847/1520 (55%)
-Bad sectors: 304/1520 (20%)
+0. 0 XXXBXX...X.XXXX......?...........X...X.........X................................
+0. 1 ..X..X.X..XB.B.B........X...........X.......X...................................
+0. 2 X.XXXX.XX....XB.................X.X..X..X......X................................
+0. 3 X.X..XXXX..?XXXX..................XX.X..........................................
+0. 4 X.X..X....X.X.XX....?....?........XXXX..X.....X.................................
+0. 5 XXXX...?..X.XBX...?......C......C?.X.X...?....X..........X......................
+0. 6 XXXB.XX.XX???XXX...............CX.XXXX........X.................................
+0. 7 XX..XX.XC..?.X......B.......X...X..XX...C.......................................
+0. 8 X?.B...XXX.?..XX........X........XCXXX..X..X..X.................................
+0. 9 BB.XX.X.X.X...BX.........C.......XXX...........X.....X..........................
+0.10 BX.XX.XX.X..XX.B...X.............XXX........................................C...
+0.11 .C.X.C..BXXBXBX?X................XX..X......X...................................
+0.12 BX.XXX....BX..X......C....X......XXX.......XX..........................XXXXXXXXX
+0.13 X..BXX..X?.XX.X....X..............XXXX.X....X...............XXXXXXXXXXXXXXXXXXXX
+0.14 X...XXB..X.X..X....X...X..C........X?...........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.15 X.BX.XX.X.XXX.X...........X.....X..X..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.16 XBXX...XX.X.X.XX........B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.17 XXB..X.B....XX..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.18 XXCXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+Good sectors: 978/1520 (64%)
+Missing sectors: 503/1520 (33%)
+Bad sectors: 39/1520 (2%)
 80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
 ```
 
 This is the **sector map**, and is showing me the status of every sector it
 found on the disk. (Tracks on the X-axis, sectors on the Y-axis.) This is a
-very bad read from a [Victor 9000](victor9k.md) disk; good reads shouldn't
-look like this. A dot represents a good sector. A B is one where the CRC
-check failed; an X is one which couldn't be found at all.
+very bad read from a [Victor 9000](disk-victor9k.md) disk; good reads
+shouldn't look like this. A dot represents a good sector. A B is one where
+the CRC check failed; an X is one which couldn't be found at all; a ? is a
+sector which was found but contained no data.
 
-At the very bottom there's a summary: 24% good sectors. Let me try and improve
+At the very bottom there's a summary: 64% good sectors. Let me try and improve
 that.
 
 (You may notice the wedge of Xs in the bottom right. This is because the
 Victor 9000 uses a varying number of sectors per track; the short tracks in
 the middle of the disk store less. So, it's perfectly normal that those
 sectors are missing. This will affect the 'good sectors' score, so it's
-normal not to have 100% on this disk.)
+normal not to have 100% on this type of disk.)
+
+Clock errors
+------------
+
+When FluxEngine sees a track, it attempts to automatically guess the clock
+rate of the data in the track. It does this by looking for the magic bit
+pattern at the beginning of each sector record and measuring how long it
+takes. This is shown in the tracing FluxEngine produces as it runs. For
+example:
+
+```
+ 70.0: 868 ms in 427936 bytes
+       138 records, 69 sectors; 2.13us clock; 
+       logical track 70.0; 6656 bytes decoded.
+ 71.0: 890 ms in 387904 bytes
+       130 records, 65 sectors; 2.32us clock; 
+       logical track 71.0; 6144 bytes decoded.
+```
+
+Bits are then found by measuring the interval between pulses on the disk and
+comparing to this clock rate.
+
+However, floppy disk drives are extremely analogue devices and not necessarily calibrated very well, and the disk may be warped, or the rubber band which makes the drive work may have lost its bandiness, and so the bits are not necessarily precisely aligned. Because of this, FluxEngine can tolerate a certain amount of error. This is controlled by the `--bit-error-threshold` parameter. Varying this can have magical effects. For example, adding `--bit-error-threshold=0.4` turns the decode into this:
+
+```
+H.SS Tracks --->
+0. 0 ...B............................................................................
+0. 1 ...........B...B................................................................
+0. 2 ..............B.................................................................
+0. 3 ................................................................................
+0. 4 ................................................................................
+0. 5 ...B............................................................................
+0. 6 ...B.....B......................................................................
+0. 7 ...B...........B....B...........................................................
+0. 8 ................................................................................
+0. 9 .B............B.................................................................
+0.10 .B............BB................................................................
+0.11 B............B..................................................................
+0.12 ...B......B............................................................XXXXXXXXX
+0.13 ............B...............................................XXXXXXXXXXXXXXXXXXXX
+0.14 ...B......B.....................................XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.15 ..B.....BB..B.........................XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.16 ..B.....B.B.............B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.17 ..B....B........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0.18 .B..XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+Good sectors: 1191/1520 (78%)
+Missing sectors: 296/1520 (19%)
+Bad sectors: 33/1520 (2%)
+80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
+```
+
+A drastic difference!
+
+The value of the parameter is the fraction of a clock of error to accept. The
+value typically varies from 0.0 to 0.5; the default is 0.2. Larger values
+make FluxEngine more tolerant, so trying 0.4 is the first thing to do when
+faced with a dubious disk. However, in some situations, increasing the value
+can actually _increase_ the error rate --- which is why 0.4 isn't the default
+--- so you'll need to experiment.
+
+That's the most common tuning parameter, but others are available:
+
+`--pulse-debounce-threshold` controls whether FluxEngine ignores pairs of pulses in rapid succession. This is common on some disks (I've observed them on Brother word processor disks).
+
+`--clock-interval-bias` adds a constant bias to the intervals between pulses
+before doing decodes. This is very occasionally necessary to get clean reads
+--- for example, if the machine which wrote the disk always writes pulses
+late. If you try this, use very small numbers (e.g. 0.02). Negative values
+are allowed.
+
+Both these parameters take a fraction of a clock as a parameter, and you'll
+probably never need to touch them.
 
 Clock detection
 ---------------
 
-When FluxEngine sees a track, it attempts to automatically guess the clock
-rate of the data in the track. It does this by computing a histogram of the
-spacing between pulses and attempting to detect the shortest peak. The
-histogram should look something like this.
+A very useful tool for examining problematic disks is `fluxengine inspect`.
+This will let you examine the raw flux on a disk (or flux file). It'll also
+guess the clock rate on the disk for you, using a simple statistical analysis
+of the pulse intervals on the disk. (Note that the tool only works on one
+track at a time.)
 
 ```
+$ fluxengine inspect -s good.flux:t=0:s=0
 Clock detection histogram:
 3.58    737 ▉
 3.67   3838 ████▊
@@ -96,6 +178,7 @@ Signal level: 3170
 Peak start:   42 (3.50 us)
 Peak end:     52 (4.33 us)
 Median:       47 (3.92 us)
+3.92us clock detected.
 ```
 
 That's _not_ the histogram from the Victor disk; that's an Apple II disk, and
@@ -109,190 +192,65 @@ So, what does my Victor 9000 histogram look like? Let's look at the
 histogram for a single track:
 
 ```
-$ fe-readvictor -s diskimage/:s=0:t=0 --show-clock-histogram
-Reading from: diskimage/:d=0:s=0:t=0
-  0.0: 829 ms in 316193 bytes
+$ fluxengine inspect -s dubious.flux:t=0:s=0
 Clock detection histogram:
-1.25    447 ▌
-1.33  16283 ████████████████████▎
-1.42  22879 ████████████████████████████▌
-1.50   4564 █████▋
-1.58   1272 █▌
-1.67  19594 ████████████████████████▍
-1.75  32059 ████████████████████████████████████████ 
-1.83  18042 ██████████████████████▌
-1.92   2249 ██▊
-2.00   8825 ███████████ 
-2.08  16031 ████████████████████ 
-2.17   5080 ██████▎
-2.25    409 ▌
-2.33   7216 █████████ 
-2.42   6269 ███████▊
-2.50    514 ▋
-2.58   5176 ██████▍
-2.67  13080 ████████████████▎
-2.75   6774 ████████▍
-2.83   1916 ██▍
-2.92  10880 █████████████▌
-3.00  21277 ██████████████████████████▌
-3.08  11625 ██████████████▌
-3.17   1028 █▎
+1.33   1904 █▉
+1.42  21669 ██████████████████████▌
+1.50   2440 ██▌
+1.58    469 ▍
+1.67   7261 ███████▌
+1.75   6808 ███████ 
+1.83   3088 ███▏
+1.92   2836 ██▉
+2.00   8897 █████████▎
+2.08   6200 ██████▍
 ...
-3.67   1910 ██▍
-3.75  19720 ████████████████████████▌
-3.83  12365 ███████████████▍
-3.92    814 █ 
-4.00   3144 ███▉
-4.08   5776 ███████▏
-4.17   3278 ████ 
-4.25   8487 ██████████▌
-4.33  15922 ███████████████████▊
-4.42   9656 ████████████ 
-4.50   1540 █▉
+2.25    531 ▌
+2.33   2802 ██▉
+2.42   2136 ██▏
+2.50   1886 █▉
+2.58  10110 ██████████▌
+2.67   8283 ████████▌
+2.75   7779 ████████ 
+2.83   2680 ██▊
+2.92  13908 ██████████████▍
+3.00  38431 ████████████████████████████████████████ 
+3.08  35708 █████████████████████████████████████▏
+3.17   5361 █████▌
 ...
-Noise floor:  320
-Signal level: 3205
-Peak start:   14 (1.17 us)
-Peak end:     39 (3.25 us)
-Median:       23 (1.92 us)
+3.75    294 ▎
+3.83    389 ▍
+3.92   1224 █▎
+4.00   3067 ███▏
+4.08   4092 ████▎
+4.17   6916 ███████▏
+4.25  25639 ██████████████████████████▋
+4.33  31407 ████████████████████████████████▋
+4.42  10209 ██████████▋
+4.50   1159 █▏
+...
+Noise floor:  384
+Signal level: 1921
+Peak start:   15 (1.25 us)
+Peak end:     26 (2.17 us)
+Median:       20 (1.67 us)
+1.67 us clock detected.
 ```
 
 That's... not good. The disk is very noisy, and the intervals between pulses
-are horribly distributed. The detected clock is 1.92us, which is clearly
-wrong.
+are horribly distributed. The detected clock from the decode is 1.45us, which
+does correspond more-or-less to a peak. You'll also notice that the
+double-clock interval is at 3.00us, which is _not_ twice 1.45us. The guessed
+clock by `fluxengine inspect` is 1.67us, which is clearly wrong.
 
-I can override the clock detection and specify the clock manually. 1.75us
-looks like a good candidate. Let's try that on track 0.
+This demonstrates that the statistical clock guessing isn't brilliant, which
+is why I've just rewritten the decoder not to use it; nevertheless, it's a
+useful tool for examining disks.
 
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0 --manual-clock-rate-us=1.75
-...skipped...
-No sectors in output; skipping analysis
-0 tracks, 0 heads, 0 sectors, 0 bytes per sector, 0 kB total
-```
+`fluxengine inspect` will also dump the raw flux data in various formats, but
+that's mostly only useful to me. Try `--dump-bits` to see the raw bit pattern
+on the disk (using the guessed clock, or `--manual-clock-rate-us` to set it
+yourself); `--dump-flux` will show you discrete pulses and the intervals
+between them.
 
-Nope, nothing --- FluxEngine was unable to find any valid data. How about
-1.42us, this time for the whole disk?
-
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0 --manual-clock-rate-us=1.42
-...skipped...
-H.SS Tracks --->
-0. 0 .BBBBBBBBXBBBBBBXXXXXXXXXBB
-0. 1 B.BBBBBBBXXXXBBBXXXXXXXXXXX
-0. 2 B..BBBBBBBBBXBBBXXXXXXXXXXX
-0. 3 B.BBBBBBBXBB.BBBXXXXXXXXXXX
-0. 4 B.BBBBBBBXBB.BBBXBXXXXXXXBB
-0. 5 B.BBBBBBBBBB.BXBXXXXXXXXXBB
-0. 6 B..BBBBBBXBBXBXBXXXXXXXXXXX
-0. 7 B..BBBBBBBBBXB.XXXBBXXXBXBX
-0. 8 B.BBBBBBBBBBXB.BXBBXXXX.XXX
-0. 9 B.BBBBBBBXXXXX.BXXXXXXXXXXX
-0.10 B..BBBBBBBBBXB.BXXXXXXXXXXX
-0.11 B..BBBBBXXBXBB.BXXXXXXXXXXX
-0.12 B.BBBBBB.BXBBB.BXXXXXXXXXXX
-0.13 B..BBBBB.BBBBB.BXXBXXXXXXXX
-0.14 BB.BBBBBXBBBBB.BXBXXXXXXXXX
-0.15 B..BBBBB.XBBBB.BXXXXXXXXXXX
-0.16 B..BBBBBBXBBXBBBXBBXXXXXXXX
-0.17 BBBBBBBB.XBBXBBBXXXXXXXXXXX
-0.18 BBBBXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 42/513 (8%)
-Missing sectors: 234/513 (45%)
-Bad sectors: 237/513 (46%)
-27 tracks, 1 heads, 19 sectors, 512 bytes per sector, 256 kB total
-```
-
-It found something. The sectors in track 0 are now B rather than X, which
-means that FluxEngine at least found them, and look, one sector even passed
-its CRC!
-
-But it turns out that the Victor 9000 actually uses a varying clock rate from
-track to track, so as to fit more data on the longer tracks on the outside of
-the disk. So, manually setting the clock rate to 1.42us has actually made
-things _worse_ at the other end of the disk. Our overall bad sector rate has
-gone up from 20% to 46%.
-
-So, I look back at the histogram. I want to keep using the clock
-autodetection, but persuade it to detect the right clock. There _is_ a peak
-at 1.42us, but there's enough noise around it to confuse the peak detection
-algorithm. You can see from the summary at the end that it thinks the peak
-extends from 1.17us to 3.25us.
-
-The way to correct this is to change the noise floor. This makes it ignore
-frequencies below a certain level. Raising this will make it much more
-conservative about what it considers a frequency peak. With good data, this
-actually makes frequency detection _less_ accurate, but with bad data it can
-help.
-
-```
-$ fe-readvictor9k -s diskimage/:s=0:t=0  --show-clock-histogram --noise-floor-factor=0.25
-Reading from: diskimage/:d=0:s=0:t=0
-  0.0: 829 ms in 316193 bytes
-Clock detection histogram:
-1.33  16283 ████████████████████▎
-1.42  22879 ████████████████████████████▌
-1.50   4564 █████▋
-...
-1.67  19594 ████████████████████████▍
-1.75  32059 ████████████████████████████████████████ 
-1.83  18042 ██████████████████████▌
-...
-2.00   8825 ███████████ 
-2.08  16031 ████████████████████ 
-2.17   5080 ██████▎
-...
-...skipped...
-Noise floor:  8014
-Signal level: 3205
-Peak start:   15 (1.25 us)
-Peak end:     18 (1.50 us)
-Median:       17 (1.42 us)
-...skipped...
-Good sectors: 1/19 (5%)
-Missing sectors: 0/19 (0%)
-Bad sectors: 18/19 (94%)
-1 tracks, 1 heads, 19 sectors, 512 bytes per sector, 9 kB total
-```
-
-So, it's now found a peak from 1.25us to 1.50us, with a median of 1.42us ---
-exactly what I wanted. Let's try it on the whole disk:
-
-```
-H.SS Tracks --->
-0. 0 .BBBBBBBBBBBBBBBB.BBB.B..BB.....................................................
-0. 1 B.BBBBBBBBXXX.BBB.BBB.X..BB..........B..........................................
-0. 2 B..BBBBBB.BBXBBBB.BBB.X..BB.......B.............................................
-0. 3 B.BBBBBBBBBB.B.BX.BBB.B..BB......B..............................................
-0. 4 B.BBBBBB.BBB.BBBB.BBB.B..BB.....B...............................................
-0. 5 B.BBBBBBBBBB.BBBB.BBB.XB.BB.....B.B.............................................
-0. 6 B..BBBBBBBBBXB.BX.BBB.X.XXB.....................................................
-0. 7 B..BBBBBBBBBXB.XB.BBB.X.BBB.....................................................
-0. 8 B.BBBBBBBBBBXB.BB.BBB.X.XXX.B...................................................
-0. 9 B.BBBBBBBBXXX..BX.BXB.X.XXXBB...................................................
-0.10 B..BBBBB.BBBXB.BB.BBB...BBB.B...................................................
-0.11 B..BBBBB.BBXBB.BB.BXB.B.BBB.B......B............................................
-0.12 B.BBBBBB.BXBBB.BB.BXX.X.XXX........B...................................XXXXXXXXX
-0.13 B..BBBBB.BBBBB.BX.BBB.X.BXB......BB.........................XXXXXXXXXXXXXXXXXXXX
-0.14 BB.BBBBB..BBBB.BB.BBB.B.XBB.B....BB.B...........XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.15 B..BBBBB.BBBBB.BB.BBB.X.XBB.B....B....XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.16 B..BBBBB.BBBXBBBB.BBB.X.BXBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.17 BBBBBBBB.BBBX.BBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-0.18 BBBBXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-Good sectors: 834/1520 (54%)
-Missing sectors: 346/1520 (22%)
-Bad sectors: 340/1520 (22%)
-80 tracks, 1 heads, 19 sectors, 512 bytes per sector, 760 kB total
-```
-
-54% good sectors --- much better! Most of the top half of the disk is reading
-flawlessly. The bottom half is still dreadful, but much better.
-
-I picked this disk as a sample because it's essentially wrecked. It's the
-worst disk image I've ever seen. Luckily I didn't scan this myself, because
-chances are it's all mouldy and would wreck my disk heads. But its very
-badness makes it a good example.
-
-(I am continually improving the clock detection and data extraction
-algorithms. I have actually seen someone get more data than I off this image,
-with a mostly-good read of track 0. I must find out their secrets...)
+The tool's subject to change without notice as I tinker with it.

doc/technical.md

@@ -13,9 +13,9 @@ Verilog turns the timer, pulse and index information into a bytecode stream
 which encodes intervals, pulses, and whether the index hole has been seen.
 
 This is then streamed back to the PC, where offline software decodes it: it
-does simple statistical analysis to guess the clock rate, then turns the
-pulses into a nice, lined up bit array and from there decodes the file system
-format.
+searches for known bit patterns (which depend on the format), uses those to
+determine the clock rate, and then turns the pulses into a nice, lined up bit
+array and from there decodes the file system format.
 
 Writing back to disk works the same way: bytes are streamed to the
 FluxEngine, where a different datapath state machine thingy (the PSoC5LP has
@@ -24,7 +24,9 @@ stream of pulses to the disk.
 
 The bytecode format represents an interval between pulses as a byte, a pulse
 as a byte, and the index hole as a byte. Timer overflows are handled by
-sending multiple intervals in a row.
+sending multiple intervals in a row. However, the USB transport applies a
+simple compression system to this in order to get the USB bandwidth down to
+something manageable.
 
 An HD floppy has a nominal pulse frequency of 500kHz, and we use a sample
 clock of 12MHz (every 83ns). This means that our 500kHz pulses will have an
@@ -75,7 +77,7 @@ second (one for each 64-byte frame) in order to transfer the data.
 
 The Atmels and STM32s I found were perfectly capable of doing the real-time
 sampling, using hand-tool assembly, but I very much doubt whether they could
-do the USB streaming as well (although I'd like to move away from the Cypress
+do the USB streaming as well (although I want to move away from the Cypress
 onto something less proprietary and easier to source, so I'd like to be
 proven wrong here).
 
@@ -114,9 +116,13 @@ admittedly expensive.)
 
   - [The TEAC FD-05HF-8830 data
     sheet](https://hxc2001.com/download/datasheet/floppy/thirdparty/Teac/TEAC%20FD-05HF-8830.pdf):
-    the technical data sheet for a representative drive. Lots of useful
+    the technical data sheet for a representative 3.5" drive. Lots of useful
     timing numbers here.
 
+  - [The Mitsubishi M4851 data
+    sheet](http://www.bitsavers.org/pdf/mitsubishi/floppy/M4851/TJ2-G30211A_M4851_DSHH_48TPI_OEM_Manual_Nov83.pdf):
+    the equivalent data sheet for a representative 5.25" drive.
+
   - [KryoFlux stream file
     documentation](https://www.kryoflux.com/download/kryoflux_stream_protocol_rev1.1.pdf):
     the format of KryoFlux stream files (partially supported by FluxEngine)

doc/using.md

@@ -46,28 +46,30 @@ In order to do anything useful, you have to plug it in to a floppy disk drive (o
   5. Connect the FluxEngine to your PC via USB --- using the little socket on
      the board, not the big programmer plug.
 
-  6. Insert a scratch disk and do `.obj/fe-rpm` from the shell. The motor
+  6. Insert a scratch disk and do `fluxengine rpm` from the shell. The motor
      should work and it'll tell you that the disk is spinning at about 300
      rpm for a 3.5" disk, or 360 rpm for a 5.25" disk. If it doesn't, please
      [get in touch](https://github.com/davidgiven/fluxengine/issues/new).
 
-  7. Do `.obj/fe-testbulktransport` from the shell. It'll measure your USB
+  7. Do `fluxengine testbulktransport` from the shell. It'll measure your USB
      bandwidth. Ideally you should be getting above 900kB/s. FluxEngine needs
      about 850kB/s, so if you're getting less than this, try a different USB
      port.
 
   8. Insert a standard PC formatted floppy disk into the drive (probably a good
-     idea to remove the old disk first). Then do `.obj/fe-readibm`. It should
-     read the disk, emitting copious diagnostics, and spit out an `ibm.img`
-     file containing the decoded disk image (either 1440kB or 720kB depending).
+     idea to remove the old disk first). Then do `fluxengine read ibm`. It
+     should read the disk, emitting copious diagnostics, and spit out an
+     `ibm.img` file containing the decoded disk image (either 1440kB or 720kB
+     depending).
 
   9. Profit!
 
 ## The programs
 
-I'm sorry to say that the programs are very badly documented --- they're
-moving too quickly for the documentation to keep up. They do all respond to
-`--help`. There are some common properties, described below.
+I'm sorry to say that the client program is very badly documented --- it's
+moving too quickly for the documentation to keep up. It does respond to
+`--help` or `help` depending on context. There are some common properties,
+described below.
 
 ### Source and destination specifiers
 
@@ -76,7 +78,7 @@ use the `--source` (`-s`) and `--dest` (`-d`) options to tell FluxEngine
 which bits of the disk you want to access. These use a common syntax:
 
 ```
-.obj/fe-readibm -s fakedisk.flux:t=0-79:s=0
+fluxengine read ibm -s fakedisk.flux:t=0-79:s=0
 ```
 
   - To access a real disk, leave out the filename (so `:t=0-79:s=0`).
@@ -110,6 +112,70 @@ sensible for the command you're using.
 **Important note:** FluxEngine _always_ uses zero-based units (even if the
 *disk format says otherwise).
 
+### Input and output specifiers
+
+These use a very similar syntax to the source and destination specifiers
+(because they're based on the same microformat library!) but are used for
+input and output _images_: i.e. nicely lined up arrays of sectors which you
+can actually do something with.
+
+Use `--input` (`-i`) or `--output` (`-o`) as appropriate to tell FluxEngine
+where you want to read from or write to. The actual format is autodetected
+based on the extension:
+
+  - `.img` or `.adf`: raw sector images in CHS order. Append
+    `:c=80:h=2:s=9:b=512` to set the geometry; that specifies 80 cylinders, 2
+    heads, 9 sectors, 512 bytes per sector. For output files (`--output`) the
+    geometry will be autodetected if left unspecified. For input files you
+    normally have to specify it.
+
+  - `.ldbs`: John Elliott's [LDBS disk image
+    format](http://www.seasip.info/Unix/LibDsk/ldbs.html), which is
+    consumable by the [libdsk](http://www.seasip.info/Unix/LibDsk/) suite of
+    tools. This allows things like variable numbers of sectors per track
+    (e.g. Macintosh or Commodore 64) and also provides information about
+    whether sectors were read correctly. You can use libdsk to convert this
+    to other formats, using a command like this:
+
+    ```
+    $ dsktrans out.ldbs -otype tele out.td0
+    ```
+
+    ...to convert to TeleDisk format. (Note you have to use dsktrans rather
+    than dskconv due to a minor bug in the geometry hadnling.)
+
+    FluxEngine's LDBS support is currently limited to write only, and
+    it doesn't store a lot of the more esoteric LDBS features like format
+    types, timings, and data rates.
+
+  - `.d64`: the venerable Commodore 64 disk image format as used by the 1540,
+    1541, etc. This is a special-purpose format due to the weird layout of
+    1540 disks and while you can use this for non-Commodore disks the result
+    will be gibberish. Use this to image Commodore 64 disks and load the
+    result into an emulator.
+
+    FluxEngine's D64 support is currently limited to write only. It will work
+    with up to 40 logical tracks.
+
+### High density disks
+
+High density disks use a different magnetic medium to low and double density
+disks, and have different magnetic properties. 3.5" drives can usually
+autodetect what kind of medium is inserted into the drive based on the hole
+in the disk casing, but 5.25" drives can't. As a result, you need to
+explicitly tell FluxEngine on the command line whether you're using a high
+density disk or not with the `--hd` flag.
+**If you don't do this, your disks may not read correctly and will _certainly_
+fail to write correctly.**
+
+You can distinguish high density 5.25" floppies from the presence of a
+traction ring around the hole in the middle of the disk; if the ring is not
+present, the disk is probably high density. However, this isn't always the
+case, and reading the disk label is much more reliable.
+
+[Lots more information on high density vs double density disks can be found
+here.](http://www.retrotechnology.com/herbs_stuff/guzis.html)
+
 ### The commands
 
 The FluxEngine client software is a largely undocumented set of small tools.
@@ -117,43 +183,47 @@ You'll have to play with them. They all support `--help`. They're not
 installed anywhere and after building you'll find them in the `.obj`
 directory.
 
-  - `fe-erase`: wipes (all or part of) a disk --- erases it without writing
-    a pulsetrain.
+  - `fluxengine erase`: wipes (all or part of) a disk --- erases it without
+  writing a pulsetrain.
 
-  - `fe-inspect`: dumps the raw pulsetrain / bitstream to stdout. Mainly useful
-    for debugging.
+  - `fluxengine inspect`: dumps the raw pulsetrain / bitstream to stdout.
+  Mainly useful for debugging.
 
-  - `fe-read*`: reads various formats of disk. See the per-format documentation
-    linked from the table above. These all take an optional `--write-flux`
-    option which will cause the raw flux to be written to the specified file.
+  - `fluxengine read*`: reads various formats of disk. See the per-format
+  documentation linked from the table above. These all take an optional
+  `--write-flux` option which will cause the raw flux to be written to the
+  specified file.
 
-  - `fe-write*`: writes various formats of disk. Again, see the per-format
-    documentation above.
+  - `fluxengine write*`: writes various formats of disk. Again, see the
+  per-format documentation above.
 
-  - `fe-writeflux`: writes raw flux files. This is much less useful than you
-    might think: you can't write flux files read from a disk to another disk.
-    (See the [FAQ](faq.md) for more information.) It's mainly useful for flux
-    files synthesised by the other `fe-write*` commands.
+  - `fluxengine writeflux`: writes raw flux files. This is much less useful
+  than you might think: you can't write flux files read from a disk to
+  another disk. (See the [FAQ](faq.md) for more information.) It's mainly
+  useful for flux files synthesised by the other `fluxengine write` commands.
 
-  - `fe-writetestpattern`: writes regular pulses (at a configurable interval)
-    to the disk. Useful for testing drive jitter, erasing disks in a more
-    secure fashion, or simply debugging. Goes well with `fe-inspect`.
+  - `fluxengine writetestpattern`: writes regular pulses (at a configurable
+  interval) to the disk. Useful for testing drive jitter, erasing disks in a
+  more secure fashion, or simply debugging. Goes well with `fluxengine
+  inspect`.
 
-  - `fe-rpm`: measures the RPM of the drive (requires a disk in the drive).
-    Mainly useful for testing.
+  - `fluxengine rpm`: measures the RPM of the drive (requires a disk in the
+  drive). Mainly useful for testing.
 
-  - `fe-seek`: moves the head. Mainly useful for finding out whether your drive
-    can seek to track 82. (Mine can't.)
+  - `fluxengine seek`: moves the head. Mainly useful for finding out whether
+  your drive can seek to track 82. (Mine can't.)
 
-  - `fe-testbulktransport`: measures your USB throughput. You need about 600kB/s
-    for FluxEngine to work. You don't need a disk in the drive for this one.
+  - `fluxengine testbulktransport`: measures your USB throughput. You need
+  about 600kB/s for FluxEngine to work. You don't need a disk in the drive
+  for this one.
 
-  - `fe-upgradefluxfile`: occasionally I need to upgrade the flux file format in
-    a non-backwards-compatible way; this tool will upgrade flux files to the new
-    format.
+  - `fluxengine upgradefluxfile`: occasionally I need to upgrade the flux
+  file format in a non-backwards-compatible way; this tool will upgrade flux
+  files to the new format.
 
-Commands which normally take `--source` or `--dest` get a sensible default if left
-unspecified. `fe-readibm` on its own will read drive 0 and write an `ibm.img` file.
+Commands which normally take `--source` or `--dest` get a sensible default if
+left unspecified. `fluxengine read ibm` on its own will read drive 0 and
+write an `ibm.img` file.
 
 ## Extra programs
 
@@ -172,19 +242,27 @@ So you've just received, say, a huge pile of old Brother word processor disks co
 Typically I do this:
 
 ```
-$ fe-readbrother -s :d=0 -o brother.img --write-flux=brother.flux
+$ fluxengine read brother -s :d=0 -o brother.img --write-flux=brother.flux
 ```
 
-This will read the disk in drive 0 and write out a filesystem image. It'll also copy the flux to brother.flux. If I then need to tweak the settings, I can rerun the decode without having to physically touch the disk like this:
+This will read the disk in drive 0 and write out a filesystem image. It'll
+also copy the flux to brother.flux. If I then need to tweak the settings, I
+can rerun the decode without having to physically touch the disk like this:
 
 ```
-$ fe-readbrother -s brother.flux -o brother.img
+$ fluxengine read brother -s brother.flux -o brother.img
 ```
 
-If the disk is particularly fragile, you can force FluxEngine not to retry
+Apart from being drastically faster, this avoids touching the (potentially
+physically fragile) disk.
+
+If the disk is particularly dodgy, you can force FluxEngine not to retry
 failed reads with `--retries=0`. This reduces head movement. **This is not
 recommended.** Floppy disks are inherently unreliable, and the occasional bit
-error is perfectly normal; the sector will read fine next time. If you
-prevent retries, then not only do you get bad sectors in the resulting image,
-but the flux file itself contains the bad read, so attempting a decode of it
-will just reproduce the same bad data.
+error is perfectly normal; FluxEngine will retry and the sector will read
+fine next time. If you prevent retries, then not only do you get bad sectors
+in the resulting image, but the flux file itself contains the bad read, so
+attempting a decode of it will just reproduce the same bad data.
+
+See also the [troubleshooting page](problems.md) for more information about
+reading dubious disks.

lib/aeslanier/aeslanier.h

@@ -1,21 +0,0 @@
-#ifndef AESLANIER_H
-#define AESLANIER_H
-
-#define AESLANIER_RECORD_SEPARATOR 0x55555122
-#define AESLANIER_SECTOR_LENGTH    256
-
-class Sector;
-class Fluxmap;
-
-class AesLanierDecoder : public AbstractSoftSectorDecoder
-{
-public:
-    virtual ~AesLanierDecoder() {}
-
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-	nanoseconds_t guessClock(Fluxmap& fluxmap) const;
-    int recordMatcher(uint64_t fifo) const;
-};
-
-#endif

lib/aeslanier/decoder.cc

@@ -1,83 +0,0 @@
-#include "globals.h"
-#include "decoders.h"
-#include "aeslanier.h"
-#include "crc.h"
-#include "fluxmap.h"
-#include "sector.h"
-#include "bytes.h"
-#include "record.h"
-#include "fmt/format.h"
-#include <string.h>
-
-/* This is actually M2FM, rather than MFM, but it our MFM/FM decoder copes fine with it. */
-
-static Bytes reverse_bits(const Bytes& input)
-{
-    Bytes output;
-    ByteWriter bw(output);
-
-    for (uint8_t b : input)
-        bw.write_8(reverse_bits(b));
-    return output;
-}
-
-SectorVector AesLanierDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decodeFmMfm(rawdata);
-        const auto& reversed = reverse_bits(bytes);
-
-        if (reversed.size() < 0x103)
-            continue;
-
-        unsigned track = reversed[1];
-        unsigned sectorid = reversed[2];
-
-        /* Basic sanity checking. */
-
-        if (track > 85)
-            continue;
-        if (sectorid > 35)
-            continue;
-
-        /* Check header 'checksum' (which seems far too simple to mean much). */
-
-        {
-            uint8_t wanted = reversed[3];
-            uint8_t got = reversed[1] + reversed[2];
-            if (wanted != got)
-                continue;
-        }
-
-        /* Check data checksum, which also includes the header and is
-         * significantly better. */
-
-        Bytes payload = reversed.slice(1, AESLANIER_SECTOR_LENGTH);
-        uint16_t wanted = reversed.reader().seek(0x101).read_le16();
-        uint16_t got = crc16ref(MODBUS_POLY_REF, payload);
-        int status = (wanted == got) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-        auto sector = std::unique_ptr<Sector>(
-            new Sector(status, track, 0, sectorid, payload));
-        sectors.push_back(std::move(sector));
-	}
-
-	return sectors;
-}
-
-nanoseconds_t AesLanierDecoder::guessClock(Fluxmap& fluxmap) const
-{
-    return fluxmap.guessClock() / 2;
-}
-
-int AesLanierDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffffffff;
-    if (masked == AESLANIER_RECORD_SEPARATOR)
-	 	return 16;
-    return 0;
-}

lib/amiga/amiga.h

@@ -1,20 +0,0 @@
-#ifndef AMIGA_H
-#define AMIGA_H
-
-#define AMIGA_SECTOR_RECORD 0xaaaa44894489LL
-
-class Sector;
-class Fluxmap;
-
-class AmigaDecoder : public AbstractSoftSectorDecoder
-{
-public:
-    virtual ~AmigaDecoder() {}
-
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-	nanoseconds_t guessClock(Fluxmap& fluxmap) const;
-    int recordMatcher(uint64_t fifo) const;
-};
-
-#endif

lib/amiga/decoder.cc

@@ -1,111 +0,0 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "protocol.h"
-#include "record.h"
-#include "decoders.h"
-#include "sector.h"
-#include "amiga.h"
-#include "bytes.h"
-#include "fmt/format.h"
-#include <string.h>
-#include <algorithm>
-
-/* 
- * Amiga disks use MFM but it's not quite the same as IBM MFM. They only use
- * a single type of record with a different marker byte.
- * 
- * See the big comment in the IBM MFM decoder for the gruesome details of how
- * MFM works.
- */
-         
-static Bytes deinterleave(const uint8_t*& input, size_t len)
-{
-    assert(!(len & 1));
-    const uint8_t* odds = &input[0];
-    const uint8_t* evens = &input[len/2];
-    Bytes output;
-    ByteWriter bw(output);
-
-    for (size_t i=0; i<len/2; i++)
-    {
-        uint8_t o = *odds++;
-        uint8_t e = *evens++;
-
-        /* This is the 'Interleave bits with 64-bit multiply' technique from
-         * http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
-         */
-        uint16_t result =
-            (((e * 0x0101010101010101ULL & 0x8040201008040201ULL)
-                * 0x0102040810204081ULL >> 49) & 0x5555) |
-            (((o * 0x0101010101010101ULL & 0x8040201008040201ULL)
-                * 0x0102040810204081ULL >> 48) & 0xAAAA);
-        
-        bw.write_be16(result);
-    }
-
-    input += len;
-    return output;
-}
-
-static uint32_t checksum(const Bytes& bytes)
-{
-    ByteReader br(bytes);
-    uint32_t checksum = 0;
-
-    assert((bytes.size() & 3) == 0);
-    while (!br.eof())
-        checksum ^= br.read_be32();
-
-    return checksum & 0x55555555;
-}
-
-SectorVector AmigaDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& rawbytes = toBytes(rawdata);
-        const auto& bytes = decodeFmMfm(rawdata);
-
-        if (bytes.size() < 544)
-            continue;
-
-        const uint8_t* ptr = bytes.begin() + 4;
-
-        Bytes header = deinterleave(ptr, 4);
-        Bytes recoveryinfo = deinterleave(ptr, 16);
-
-        uint32_t wantedheaderchecksum = deinterleave(ptr, 4).reader().read_be32();
-        uint32_t gotheaderchecksum = checksum(rawbytes.slice(8, 40));
-        if (gotheaderchecksum != wantedheaderchecksum)
-            continue;
-
-        uint32_t wanteddatachecksum = deinterleave(ptr, 4).reader().read_be32();
-        uint32_t gotdatachecksum = checksum(rawbytes.slice(64, 1024));
-        int status = (gotdatachecksum == wanteddatachecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-        Bytes databytes = deinterleave(ptr, 512);
-        unsigned track = header[1] >> 1;
-        unsigned side = header[1] & 1;
-        auto sector = std::unique_ptr<Sector>(
-            new Sector(status, track, side, header[2], databytes));
-        sectors.push_back(std::move(sector));
-	}
-
-	return sectors;
-}
-
-nanoseconds_t AmigaDecoder::guessClock(Fluxmap& fluxmap) const
-{
-    return fluxmap.guessClock() / 2;
-}
-
-int AmigaDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint64_t masked = fifo & 0xffffffffffffULL;
-    if (masked == AMIGA_SECTOR_RECORD)
-		return 64;
-    return 0;
-}

lib/apple2/decoder.cc

@@ -1,120 +0,0 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "protocol.h"
-#include "record.h"
-#include "decoders.h"
-#include "sector.h"
-#include "apple2.h"
-#include "bytes.h"
-#include "fmt/format.h"
-#include <string.h>
-#include <algorithm>
-
-static int decode_data_gcr(uint8_t gcr)
-{
-    switch (gcr)
-    {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
-    }
-    return -1;
-};
-
-/* This is extremely inspired by the MESS implementation, written by Nathan Woods
- * and R. Belmont: https://github.com/mamedev/mame/blob/7914a6083a3b3a8c243ae6c3b8cb50b023f21e0e/src/lib/formats/ap2_dsk.cpp
- */
-static Bytes decode_crazy_data(const uint8_t* inp, int& status)
-{
-    Bytes output(APPLE2_SECTOR_LENGTH);
-
-    uint8_t checksum = 0;
-    for (unsigned i = 0; i < APPLE2_ENCODED_SECTOR_LENGTH; i++)
-    {
-        checksum ^= decode_data_gcr(*inp++);
-
-        if (i >= 86)
-        {
-            /* 6 bit */
-            output[i - 86] |= (checksum << 2);
-        }
-        else
-        {
-            /* 3 * 2 bit */
-            output[i + 0] = ((checksum >> 1) & 0x01) | ((checksum << 1) & 0x02);
-            output[i + 86] = ((checksum >> 3) & 0x01) | ((checksum >> 1) & 0x02);
-            if ((i + 172) < APPLE2_SECTOR_LENGTH)
-                output[i + 172] = ((checksum >> 5) & 0x01) | ((checksum >> 3) & 0x02);
-        }
-    }
-
-    checksum &= 0x3f;
-    uint8_t wantedchecksum = decode_data_gcr(*inp);
-    status = (checksum == wantedchecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-    return output;
-}
-
-uint8_t combine(uint16_t word)
-{
-    return word & (word >> 7);
-}
-
-SectorVector Apple2Decoder::decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-    int nextTrack;
-    int nextSector;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const std::vector<bool>& rawdata = rawrecord->data;
-        const Bytes& rawbytes = toBytes(rawdata);
-        ByteReader br(rawbytes);
-
-        if (rawbytes.size() < 8)
-            continue;
-
-        uint32_t signature = br.read_be24();
-        switch (signature)
-        {
-            case APPLE2_SECTOR_RECORD:
-            {
-                uint8_t volume = combine(br.read_be16());
-                nextTrack = combine(br.read_be16());
-                nextSector = combine(br.read_be16());
-                uint8_t checksum = combine(br.read_be16());
-                headerIsValid = checksum == (volume ^ nextTrack ^ nextSector);
-                break;
-            }
-
-            case APPLE2_DATA_RECORD:
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-                    
-                Bytes clipped_bytes = rawbytes.slice(0, APPLE2_ENCODED_SECTOR_LENGTH + 5);
-                int status = Sector::BAD_CHECKSUM;
-                auto data = decode_crazy_data(&clipped_bytes[3], status);
-
-                auto sector = std::unique_ptr<Sector>(
-                    new Sector(status, nextTrack, 0, nextSector, data));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
-}
-
-int Apple2Decoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffffff;
-    if ((masked == APPLE2_SECTOR_RECORD) || (masked == APPLE2_DATA_RECORD))
-		return 24;
-    return 0;
-}

lib/brother/brother.h

@@ -1,29 +0,0 @@
-#ifndef BROTHER_H
-#define BROTHER_H
-
-/* Brother word processor format (or at least, one of them) */
-
-#define BROTHER_SECTOR_RECORD 0xFFFFFD57
-#define BROTHER_DATA_RECORD   0xFFFFFDDB
-#define BROTHER_DATA_RECORD_PAYLOAD  256
-#define BROTHER_DATA_RECORD_CHECKSUM 3
-
-class Sector;
-class Fluxmap;
-
-class BrotherDecoder : public AbstractSoftSectorDecoder
-{
-public:
-    virtual ~BrotherDecoder() {}
-
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
-};
-
-extern void writeBrotherSectorHeader(std::vector<bool>& bits, unsigned& cursor,
-		int track, int sector);
-extern void writeBrotherSectorData(std::vector<bool>& bits, unsigned& cursor,
-		const Bytes& data);
-
-#endif

lib/brother/decoder.cc

@@ -1,136 +0,0 @@
-#include "globals.h"
-#include "sql.h"
-#include "fluxmap.h"
-#include "decoders.h"
-#include "record.h"
-#include "brother.h"
-#include "sector.h"
-#include "bytes.h"
-#include "crc.h"
-#include <ctype.h>
-
-static std::vector<uint8_t> outputbuffer;
-
-/*
- * Brother disks have this very very non-IBM system where sector header records
- * and data records use two different kinds of GCR: sector headers are 8-in-16
- * (but the encodable values range from 0 to 77ish only) and data headers are
- * 5-in-8. In addition, there's a non-encoded 10-bit ID word at the beginning
- * of each record, as well as a string of 53 1s introducing them. That does at
- * least make them easy to find.
- *
- * Disk formats vary from machine to machine, but mine uses 78 tracks. Track 0
- * is erased but not formatted.  Track alignment is extremely dubious and
- * Brother track 0 shows up on my machine at track 2.
- */
-
-static int decode_data_gcr(uint8_t gcr)
-{
-    switch (gcr)
-    {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
-    }
-    return -1;
-};
-
-static int decode_header_gcr(uint16_t word)
-{
-	switch (word)
-	{
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "header_gcr.h"
-		#undef GCR_ENTRY
-	}                       
-	return -1;             
-};
-
-SectorVector BrotherDecoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-	bool headerIsValid = false;
-	unsigned nextTrack = 0;
-	unsigned nextSector = 0;
-
-    for (auto& rawrecord : rawRecords)
-    {
-		if (rawrecord->data.size() < 64)
-		{
-			headerIsValid = false;
-			continue;
-		}
-
-		auto ii = rawrecord->data.cbegin();
-		uint32_t signature = toBytes(ii, ii+32).reader().read_be32();
-		switch (signature)
-		{
-			case BROTHER_SECTOR_RECORD:
-			{
-				headerIsValid = false;
-				if (rawrecord->data.size() < (32+32))
-					break;
-
-				const auto& by = toBytes(ii+32, ii+64);
-				ByteReader br(by);
-				nextTrack = decode_header_gcr(br.read_be16());
-				nextSector = decode_header_gcr(br.read_be16());
-
-				/* Sanity check the values read; there's no header checksum and
-				 * occasionally we get garbage due to bit errors. */
-				if (nextSector > 11)
-					break;
-				if (nextTrack > 79)
-					break;
-
-				headerIsValid = true;
-				break;
-			}
-
-			case BROTHER_DATA_RECORD:
-			{
-				if (!headerIsValid)
-					break;
-
-				Bytes rawbytes = toBytes(rawrecord->data.cbegin()+32, rawrecord->data.cend());
-				const int totalsize = BROTHER_DATA_RECORD_PAYLOAD + BROTHER_DATA_RECORD_CHECKSUM;
-
-				Bytes output;
-				ByteWriter bw(output);
-				BitWriter bitw(bw);
-				for (uint8_t b : rawbytes)
-				{
-					uint32_t nibble = decode_data_gcr(b);
-					bitw.push(nibble, 5);
-					if (output.size() == totalsize)
-						break;
-				}
-				bitw.flush();
-				output.resize(totalsize);
-
-				Bytes payload = output.slice(0, BROTHER_DATA_RECORD_PAYLOAD);
-				uint32_t realCrc = crcbrother(payload);
-				uint32_t wantCrc = output.reader().seek(BROTHER_DATA_RECORD_PAYLOAD).read_be24();
-				int status = (realCrc == wantCrc) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                headerIsValid = false;
-				break;
-			}
-		}
-	}
-
-	return sectors;
-}
-
-int BrotherDecoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffffffff;
-	if ((masked == BROTHER_SECTOR_RECORD) || (masked == BROTHER_DATA_RECORD))
-		return 32;
-    return 0;
-}

lib/brother/encoder.cc

@@ -1,92 +0,0 @@
-#include "globals.h"
-#include "record.h"
-#include "decoders.h"
-#include "brother.h"
-#include "crc.h"
-
-static int encode_header_gcr(uint16_t word)
-{
-	switch (word)
-	{
-		#define GCR_ENTRY(gcr, data) \
-			case data: return gcr;
-		#include "header_gcr.h"
-		#undef GCR_ENTRY
-	}                       
-	return -1;             
-};
-
-static int encode_data_gcr(uint8_t data)
-{
-	switch (data)
-	{
-		#define GCR_ENTRY(gcr, data) \
-			case data: return gcr;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
-	}                       
-	return -1;             
-};
-
-static void write_bits(std::vector<bool>& bits, unsigned& cursor, uint32_t data, int width)
-{
-	cursor += width;
-	for (int i=0; i<width; i++)
-	{
-		unsigned pos = cursor - i - 1;
-		if (pos < bits.size())
-			bits[pos] = data & 1;
-		data >>= 1;
-	}
-}
-
-void writeBrotherSectorHeader(std::vector<bool>& bits, unsigned& cursor,
-		int track, int sector)
-{
-	write_bits(bits, cursor, 0xffffffff, 31);
-	write_bits(bits, cursor, BROTHER_SECTOR_RECORD, 32);
-	write_bits(bits, cursor, encode_header_gcr(track), 16);
-	write_bits(bits, cursor, encode_header_gcr(sector), 16);
-	write_bits(bits, cursor, encode_header_gcr(0x2f), 16);
-}
-
-void writeBrotherSectorData(std::vector<bool>& bits, unsigned& cursor, const Bytes& data)
-{
-	write_bits(bits, cursor, 0xffffffff, 32);
-	write_bits(bits, cursor, BROTHER_DATA_RECORD, 32);
-
-	uint16_t fifo = 0;
-	int width = 0;
-
-	if (data.size() != BROTHER_DATA_RECORD_PAYLOAD)
-		Error() << "unsupported sector size";
-
-	auto write_byte = [&](uint8_t byte)
-	{
-		fifo |= (byte << (8 - width));
-		width += 8;
-
-		while (width >= 5)
-		{
-			uint8_t quintet = fifo >> 11;
-			fifo <<= 5;
-			width -= 5;
-
-			write_bits(bits, cursor, encode_data_gcr(quintet), 8);
-		}
-	};
-
-	for (uint8_t byte : data)
-		write_byte(byte);
-
-	uint32_t realCrc = crcbrother(data);
-	write_byte(realCrc>>16);
-	write_byte(realCrc>>8);
-	write_byte(realCrc);
-	write_byte(0x58); /* magic */
-    write_byte(0xd4);
-    while (width != 0)
-        write_byte(0);
-}
-
-

lib/bytes.cc

@@ -1,6 +1,8 @@
 #include "globals.h"
 #include "bytes.h"
 #include "fmt/format.h"
+#include "common/crunch.h"
+#include <fstream>
 #include <zlib.h>
 
 static std::shared_ptr<std::vector<uint8_t>> createVector(unsigned size)
@@ -124,12 +126,16 @@ uint8_t& Bytes::operator [] (unsigned pos)
 Bytes Bytes::slice(unsigned start, unsigned len) const
 {
     start += _low;
-    boundsCheck(start);
     unsigned end = start + len;
-    if (end > _high)
+    if (start >= _high)
+    {
+        /* Asking for a completely out-of-range slice --- just return zeroes. */
+        return Bytes(len);
+    }
+    else if (end > _high)
     {
         /* Can't share the buffer, as we need to zero-pad the end. */
-        Bytes b(end - start);
+        Bytes b(len);
         std::uninitialized_copy(cbegin()+start, cend(), b.begin());
         return b;
     }
@@ -163,6 +169,23 @@ Bytes toBytes(
     return bytes;
 }
 
+Bytes Bytes::swab() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    ByteReader br(*this);
+
+    while (!br.eof())
+    {
+        uint8_t a = br.read_8();
+        uint8_t b = br.read_8();
+        bw.write_8(b);
+        bw.write_8(a);
+    }
+
+    return output;
+}
+
 Bytes Bytes::compress() const
 {
     uLongf destsize = compressBound(size());
@@ -204,6 +227,70 @@ Bytes Bytes::decompress() const
     return output;
 }
 
+Bytes Bytes::crunch() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    Bytes outputBuffer(1024*1024);
+
+    crunch_state_t cs = {};
+    cs.inputptr = begin();
+    cs.inputlen = size();
+
+    do
+    {
+        cs.outputptr = outputBuffer.begin();
+        cs.outputlen = outputBuffer.size();
+
+        ::crunch(&cs);
+        bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+    }
+    while (cs.inputlen != 0);
+    cs.outputptr = outputBuffer.begin();
+    cs.outputlen = outputBuffer.size();
+    donecrunch(&cs);
+    bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+
+    return output;
+}
+
+Bytes Bytes::uncrunch() const
+{
+    Bytes output;
+    ByteWriter bw(output);
+    Bytes outputBuffer(1024*1024);
+
+    crunch_state_t cs = {};
+    cs.inputptr = begin();
+    cs.inputlen = size();
+
+    do
+    {
+        cs.outputptr = outputBuffer.begin();
+        cs.outputlen = outputBuffer.size();
+
+        ::uncrunch(&cs);
+        bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+    }
+    while (cs.inputlen != 0);
+    cs.outputptr = outputBuffer.begin();
+    cs.outputlen = outputBuffer.size();
+    doneuncrunch(&cs);
+    bw += outputBuffer.slice(0, outputBuffer.size() - cs.outputlen);
+
+    return output;
+}
+
+void Bytes::writeToFile(const std::string& filename) const
+{
+    std::ofstream f(filename, std::ios::out | std::ios::binary);
+    if (!f.is_open())
+        Error() << fmt::format("cannot open output file '{}'", filename);
+
+    f.write((const char*) cbegin(), size());
+    f.close();
+}
+
 ByteReader Bytes::reader() const
 {
     return ByteReader(*this);
@@ -214,6 +301,16 @@ ByteWriter Bytes::writer()
     return ByteWriter(*this);
 }
 
+ByteWriter& ByteWriter::operator +=(std::istream& stream)
+{
+    Bytes buffer(4096);
+
+    while (stream.read((char*) buffer.begin(), buffer.size()))
+        this->append(buffer);
+    this->append(buffer.slice(0, stream.gcount()));
+    return *this;
+}
+
 void BitWriter::push(uint32_t bits, size_t size)
 {
     bits <<= 32-size;

lib/bytes.h

@@ -43,13 +43,21 @@ public:
     void adjustBounds(unsigned pos);
     Bytes& resize(unsigned size);
 
+    Bytes& clear()
+    { resize(0); return *this; }
+
     Bytes slice(unsigned start, unsigned len) const;
+    Bytes swab() const;
     Bytes compress() const;
     Bytes decompress() const;
+    Bytes crunch() const;
+    Bytes uncrunch() const;
 
     ByteReader reader() const;
     ByteWriter writer();
 
+    void writeToFile(const std::string& filename) const;
+
 private:
     std::shared_ptr<std::vector<uint8_t>> _data;
     unsigned _low;
@@ -67,7 +75,7 @@ public:
 
     unsigned pos = 0;
     bool eof() const
-    { return pos == _bytes.size(); }
+    { return pos >= _bytes.size(); }
 
     ByteReader& seek(unsigned pos)
     {
@@ -75,6 +83,19 @@ public:
         return *this;
     }
 
+    ByteReader& skip(int delta)
+    {
+        this->pos += delta;
+        return *this;
+    }
+
+    const Bytes read(unsigned len)
+    {
+        const Bytes bytes = _bytes.slice(pos, len);
+        pos += len;
+        return bytes;
+    }
+
     uint8_t read_8()
     {
         return _bytes[pos++];
@@ -247,6 +268,18 @@ public:
         return *this;
     }
 
+    ByteWriter& operator += (std::istream& stream);
+
+    ByteWriter& append(const Bytes data)
+    {
+        return *this += data;
+    }
+
+    ByteWriter& append(std::istream& stream)
+    {
+        return *this += stream;
+    }
+
 private:
     Bytes& _bytes;
 };

lib/c64/c64.h

@@ -1,20 +0,0 @@
-#ifndef C64_H
-#define C64_H
-
-#define C64_RECORD_SEPARATOR 0xfff5
-#define C64_SECTOR_LENGTH    256
-
-class Sector;
-class Fluxmap;
-
-class Commodore64Decoder : public AbstractSoftSectorDecoder
-{
-public:
-    virtual ~Commodore64Decoder() {}
-
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
-};
-
-#endif

lib/c64/decoder.cc

@@ -1,113 +0,0 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "protocol.h"
-#include "record.h"
-#include "decoders.h"
-#include "sector.h"
-#include "c64.h"
-#include "crc.h"
-#include "bytes.h"
-#include "fmt/format.h"
-#include <string.h>
-#include <algorithm>
-
-static int decode_data_gcr(uint8_t gcr)
-{
-    switch (gcr)
-    {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
-    }
-    return -1;
-};
-
-static Bytes decode(const std::vector<bool>& bits)
-{
-    Bytes output;
-    ByteWriter bw(output);
-    BitWriter bitw(bw);
-
-    auto ii = bits.begin();
-    while (ii != bits.end())
-    {
-        uint8_t inputfifo = 0;
-        for (size_t i=0; i<5; i++)
-        {
-            if (ii == bits.end())
-                break;
-            inputfifo = (inputfifo<<1) | *ii++;
-        }
-
-        bitw.push(decode_data_gcr(inputfifo), 4);
-    }
-    bitw.flush();
-
-    return output;
-}
-
-SectorVector Commodore64Decoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-    unsigned nextSector;
-    unsigned nextTrack;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decode(rawdata);
-
-        if (bytes.size() == 0)
-            continue;
-
-        switch (bytes[0])
-        {
-            case 8: /* sector record */
-            {
-                headerIsValid = false;
-                if (bytes.size() < 6)
-                    break;
-
-                uint8_t checksum = bytes[1];
-                nextSector = bytes[2];
-                nextTrack = bytes[3] - 1;
-                if (checksum != xorBytes(bytes.slice(2, 4)))
-                    break;
-
-                headerIsValid = true;
-                break;
-            }
-            
-            case 7: /* data record */
-            {
-                if (!headerIsValid)
-                    break;
-                headerIsValid = false;
-                if (bytes.size() < 258)
-                    break;
-
-                Bytes payload = bytes.slice(1, C64_SECTOR_LENGTH);
-                uint8_t gotChecksum = xorBytes(payload);
-                uint8_t wantChecksum = bytes[257];
-                int status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
-}
-
-int Commodore64Decoder::recordMatcher(uint64_t fifo) const
-{
-    uint16_t masked = fifo & 0xffff;
-    if (masked == C64_RECORD_SEPARATOR)
-		return 4;
-    return 0;
-}

lib/common/crunch.c

@@ -0,0 +1,86 @@
+#include <stdint.h>
+#include <stdbool.h>
+#include "crunch.h"
+
+void crunch(crunch_state_t* state)
+{
+    while (state->inputlen && state->outputlen)
+    {
+        uint8_t data = *state->inputptr++;
+        state->inputlen--;
+
+        if (data & 0x80)
+        {
+            state->fifo = (state->fifo << 2) | 2 | (data & 1);
+            state->fifolen += 2;
+        }
+        else
+        {
+            state->fifo = (state->fifo << 8) | data;
+            state->fifolen += 8;
+        }
+
+        if (state->fifolen >= 8)
+        {
+            data = state->fifo >> (state->fifolen - 8);
+            *state->outputptr++ = data;
+            state->outputlen--;
+            state->fifolen -= 8;
+        }
+    }
+}
+
+void donecrunch(crunch_state_t* state)
+{
+    if (state->fifolen > 0)
+    {
+        uint8_t b = 0;
+        state->inputptr = &b;
+        state->inputlen = 1;
+        crunch(state);
+    }
+}
+
+void uncrunch(crunch_state_t* state)
+{
+    while (state->inputlen && state->outputlen)
+    {
+        if (state->fifolen < 8)
+        {
+            if (state->inputlen)
+            {
+                state->fifo = (state->fifo << 8) | *state->inputptr++;
+                state->inputlen--;
+                state->fifolen += 8;
+            }
+            else
+                state->fifo <<= 8;
+        }
+
+        uint8_t data = state->fifo >> (state->fifolen - 8);
+        if (data & 0x80)
+        {
+            data = ((data >> 6) & 0x01) | 0x80;
+            state->fifolen -= 2;
+        }
+        else
+            state->fifolen -= 8;
+
+        if (data)
+        {
+            *state->outputptr++ = data;
+            state->outputlen--;
+        }
+    }
+}
+
+void doneuncrunch(crunch_state_t* state)
+{
+    if (state->fifolen > 0)
+    {
+        uint8_t b = 0;
+        state->inputptr = &b;
+        state->inputlen = 1;
+        uncrunch(state);
+    }
+}

lib/common/crunch.h

@@ -0,0 +1,45 @@
+#ifndef CRUNCH_H
+#define CRUNCH_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/* To save bandwidth, we compress the byte stream from the sampler when
+ * sending it over USB. The encoding used is:
+ * 
+ * 0nnn.nnnn: value 0x00..0x7f
+ * 1n       : value 0x80|n
+ *
+ * The end of the buffer is terminated with zeroes, which are ignored
+ * (not written to the output).
+ * 
+ * This saves ~40%, which gets us in under the bandwidth cap.
+ */
+    
+typedef struct crunch_state_t
+{
+    const uint8_t* inputptr;
+    uint32_t inputlen;
+    uint8_t* outputptr;
+    uint32_t outputlen;
+    uint16_t fifo;
+    uint8_t fifolen;
+}
+crunch_state_t;
+
+/* Crunches as much as possible and then stops. */
+extern void crunch(crunch_state_t* state);
+extern void donecrunch(crunch_state_t* state);
+
+/* Uncrunches as much as possible and then stops. */
+extern void uncrunch(crunch_state_t* state);
+extern void doneuncrunch(crunch_state_t* state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

lib/crc.cc

@@ -2,6 +2,47 @@
 #include "bytes.h"
 #include "crc.h"
 
+template <class T>
+T reflect(T bin, unsigned width = sizeof(T)*8)
+{
+	T bout = 0;
+	while (width--)
+	{
+		bout <<= 1;
+		bout |= (bin & 1);
+		bin >>= 1;
+	}
+	return bout;
+}
+
+uint64_t generic_crc(const struct crcspec& spec, const Bytes& bytes)
+{
+	uint64_t crc = spec.init;
+	uint64_t top = 1LL << (spec.width-1);
+	uint64_t mask = (top<<1) - 1;
+
+	for (uint8_t b : bytes)
+	{
+		if (spec.refin)
+			b = reflect(b);
+
+		for (uint8_t i = 0x80; i != 0; i >>= 1)
+		{
+			uint64_t bit = crc & top;
+			crc <<= 1;
+			if (b & i)
+				bit ^= top;
+			if (bit)
+				crc ^= spec.poly;
+		}
+	}
+
+	if (spec.refout)
+		crc = reflect(crc, spec.width);
+	crc ^= spec.xorout;
+	return crc & mask;
+}
+
 uint16_t sumBytes(const Bytes& bytes)
 {
 	ByteReader br(bytes);
@@ -36,11 +77,10 @@ uint16_t crc16(uint16_t poly, uint16_t crc, const Bytes& bytes)
 	return crc;
 }
 
-uint16_t crc16ref(uint16_t poly, const Bytes& bytes)
+uint16_t crc16ref(uint16_t poly, uint16_t crc, const Bytes& bytes)
 {
 	ByteReader br(bytes);
 
-	uint16_t crc = 0xffff;
 	while (!br.eof())
 	{
 		crc ^= br.read_8();

lib/crc.h

@@ -6,14 +6,29 @@
 #define MODBUS_POLY_REF 0xa001
 #define BROTHER_POLY    0x000201
 
+struct crcspec
+{
+    unsigned width;
+    uint64_t poly;
+    uint64_t init;
+    uint64_t xorout;
+    bool refin;
+    bool refout;
+};
+
+extern uint64_t generic_crc(const struct crcspec& spec, const Bytes& bytes);
+
 extern uint16_t sumBytes(const Bytes& bytes);
 extern uint8_t xorBytes(const Bytes& bytes);
 extern uint16_t crc16(uint16_t poly, uint16_t init, const Bytes& bytes);
-extern uint16_t crc16ref(uint16_t poly, const Bytes& bytes);
+extern uint16_t crc16ref(uint16_t poly, uint16_t init, const Bytes& bytes);
 extern uint32_t crcbrother(const Bytes& bytes);
 
 static inline uint16_t crc16(uint16_t poly, const Bytes& bytes)
 { return crc16(poly, 0xffff, bytes); }
 
+static inline uint16_t crc16ref(uint16_t poly, const Bytes& bytes)
+{ return crc16ref(poly, 0xffff, bytes); }
+
 #endif
 

lib/dataspec.cc

@@ -5,8 +5,10 @@
 #include <regex>
 #include <sstream>
 
-static const std::regex MOD_REGEX("([a-z]*)=([-x+0-9,]*)");
-static const std::regex DATA_REGEX("([0-9]+)(?:(?:-([0-9]+))|(?:\\+([0-9]+)))?(?:x([0-9]+))?");
+MissingModifierException::MissingModifierException(const std::string& mod):
+    mod(mod),
+    std::runtime_error(fmt::format("missing mandatory modifier '{}'", mod))
+{}
 
 std::vector<std::string> DataSpec::split(
         const std::string& s, const std::string& delimiter)
@@ -30,6 +32,9 @@ std::vector<std::string> DataSpec::split(
 
 DataSpec::Modifier DataSpec::parseMod(const std::string& spec)
 {
+    static const std::regex MOD_REGEX("([a-z]*)=([-x+0-9,]*)");
+    static const std::regex DATA_REGEX("([0-9]+)(?:(?:-([0-9]+))|(?:\\+([0-9]+)))?(?:x([0-9]+))?");
+
     std::smatch match;
     if (!std::regex_match(spec, match, MOD_REGEX))
         Error() << "invalid data modifier syntax '" << spec << "'";
@@ -74,31 +79,108 @@ void DataSpec::set(const std::string& spec)
     filename = words[0];
     if (words.size() > 1)
     {
-        locations.clear();
-
         for (size_t i = 1; i < words.size(); i++)
         {
             auto mod = parseMod(words[i]);
-            if ((mod.name != "t") && (mod.name != "s") && (mod.name != "d"))
-                Error() << fmt::format("unknown data modifier '{}'", mod.name);
             modifiers[mod.name] = mod;
         }
+    }
+}
 
-        const auto& drives = modifiers["d"].data;
+const DataSpec::Modifier& DataSpec::at(const std::string& mod) const
+{
+    try
+    {
+        return modifiers.at(mod);
+    }
+    catch (const std::out_of_range& e)
+    {
+        throw MissingModifierException(mod);
+    }
+}
+
+bool DataSpec::has(const std::string& mod) const
+{
+    return modifiers.find(mod) != modifiers.end();
+}
+
+FluxSpec::FluxSpec(const DataSpec& spec)
+{
+    try 
+    {
+        filename = spec.filename;
+
+        locations.clear();
+
+        const auto& drives = spec.at("d").data;
         if (drives.size() != 1)
             Error() << "you must specify exactly one drive";
         drive = *drives.begin();
 
-        const auto& tracks = modifiers["t"].data;
-        const auto& sides = modifiers["s"].data;
+        const auto& tracks = spec.at("t").data;
+        const auto& sides = spec.at("s").data;
         for (auto track : tracks)
         {
             for (auto side : sides)
                 locations.push_back({ drive, track, side });
         }
+
+        for (const auto& e : spec.modifiers)
+        {
+            const auto name = e.second.name;
+            if ((name != "t") && (name != "s") && (name != "d"))
+                Error() << fmt::format("unknown fluxspec modifier '{}'", name);
+        }
+    }
+    catch (const MissingModifierException& e)
+    {
+        Error() << e.what() << " in fluxspec '" << spec << "'";
     }
 }
 
+ImageSpec::ImageSpec(const DataSpec& spec)
+{
+    try
+    {
+        filename = spec.filename;
+
+        if (!spec.has("c") && !spec.has("h") && !spec.has("s") && !spec.has("b"))
+        {
+            cylinders = heads = sectors = bytes = 0;
+            initialised = false;
+        }
+        else
+        {
+            cylinders = spec.at("c").only();
+            heads = spec.at("h").only();
+            sectors = spec.at("s").only();
+            bytes = spec.at("b").only();
+            initialised = true;
+        }
+    }
+    catch (const MissingModifierException& e)
+    {
+        Error() << e.what() << " in imagespec '" << spec << "'";
+    }
+
+    for (const auto& e : spec.modifiers)
+    {
+        const auto name = e.second.name;
+        if ((name != "c") && (name != "h") && (name != "s") && (name != "b"))
+            Error() << fmt::format("unknown fluxspec modifier '{}'", name);
+    }
+}
+
+ImageSpec::ImageSpec(const std::string filename,
+        unsigned cylinders, unsigned heads, unsigned sectors, unsigned bytes):
+    filename(filename),
+    cylinders(cylinders),
+    heads(heads),
+    sectors(sectors),
+    bytes(bytes),
+    initialised(true)
+{}
+
 DataSpec::operator std::string(void) const
 {
     std::stringstream ss;

lib/dataspec.h

@@ -1,8 +1,57 @@
 #ifndef DATASPEC_H
 #define DATASPEC_H
 
+class MissingModifierException : public std::runtime_error
+{
+public:
+    MissingModifierException(const std::string& mod);
+    const std::string mod;
+};
+
 class DataSpec
 {
+public:
+    struct Modifier
+    {
+        std::string name;
+        std::set<unsigned> data;
+        std::string source;
+
+        bool operator == (const Modifier& other) const
+        { return (name == other.name) && (data == other.data); }
+
+        bool operator != (const Modifier& other) const
+        { return (name != other.name) || (data != other.data); }
+
+        unsigned only() const
+        {
+            if (data.size() != 1)
+                Error() << "modifier " << name << " can only have one value";
+            return *(data.begin());
+        }
+    };
+
+public:
+    static std::vector<std::string> split(
+        const std::string& s, const std::string& delimiter);
+    static Modifier parseMod(const std::string& spec);
+
+public:
+    DataSpec(const std::string& spec)
+    { set(spec); }
+
+    void set(const std::string& spec);
+    operator std::string () const;
+
+    const Modifier& at(const std::string& mod) const;
+    bool has(const std::string& mod) const;
+
+    std::string filename;
+    std::map<std::string, Modifier> modifiers;
+};
+
+class FluxSpec
+{
 public:
     struct Location
     {
@@ -17,39 +66,32 @@ public:
         { return (drive != other.drive) || (track != other.track) || (side != other.side); }
     };
 
-    struct Modifier
-    {
-        std::string name;
-        std::set<unsigned> data;
-        std::string source;
-
-        bool operator == (const Modifier& other) const
-        { return (name == other.name) && (data == other.data); }
-
-        bool operator != (const Modifier& other) const
-        { return (name != other.name) || (data != other.data); }
-    };
+public:
+    FluxSpec(const DataSpec& dataspec);
 
 public:
-    static std::vector<std::string> split(
-        const std::string& s, const std::string& delimiter);
-    static Modifier parseMod(const std::string& spec);
-
-public:
-    DataSpec(const std::string& spec)
-    { set(spec); }
-
-    void set(const std::string& spec);
-    operator std::string () const;
-
     std::string filename;
-    std::map<std::string, Modifier> modifiers;
     std::vector<Location> locations;
     unsigned drive;
-    unsigned revolutions;
 };
 
-std::ostream& operator << (std::ostream& os, const DataSpec& dataSpec)
+class ImageSpec
+{
+public:
+    ImageSpec(const DataSpec& dataspec);
+    ImageSpec(const std::string filename,
+        unsigned cylinders, unsigned heads, unsigned sectors, unsigned bytes);
+
+public:
+    std::string filename;
+    unsigned cylinders;
+    unsigned heads;
+    unsigned sectors;
+    unsigned bytes;
+    bool initialised : 1;
+};
+
+static inline std::ostream& operator << (std::ostream& os, const DataSpec& dataSpec)
 { os << (std::string)dataSpec; return os; }
 
 class DataSpecFlag : public Flag
@@ -58,15 +100,21 @@ public:
     DataSpecFlag(const std::vector<std::string>& names, const std::string helptext,
             const std::string& defaultValue):
         Flag(names, helptext),
-        value(defaultValue)
+        _value(defaultValue)
     {}
 
-    bool hasArgument() const { return true; }
-    const std::string defaultValueAsString() const { return value; }
-    void set(const std::string& value) { this->value.set(value); }
+    const DataSpec& get() const
+    { checkInitialised(); return _value; }
 
-public:
-    DataSpec value;
+    operator const DataSpec& () const
+    { return get(); }
+
+    bool hasArgument() const { return true; }
+    const std::string defaultValueAsString() const { return _value; }
+    void set(const std::string& value) { _value.set(value); }
+
+private:
+    DataSpec _value;
 };
 
 #endif

lib/decoders/decoders.cc

@@ -1,317 +1,79 @@
 #include "globals.h"
 #include "flags.h"
 #include "fluxmap.h"
-#include "decoders.h"
+#include "decoders/fluxmapreader.h"
+#include "decoders/decoders.h"
 #include "record.h"
 #include "protocol.h"
-#include "rawbits.h"
+#include "decoders/rawbits.h"
+#include "track.h"
+#include "sector.h"
 #include "fmt/format.h"
 #include <numeric>
 
-static DoubleFlag pulseAccuracyFactor(
-    { "--pulse-accuracy-factor" },
-    "Pulses must be within this much of an expected clock tick to register (in clock periods).",
-    0.4);
-
-static SettableFlag showClockHistogram(
-    { "--show-clock-histogram" },
-    "Dump the clock detection histogram.");
-
-static DoubleFlag manualClockRate(
-	{ "--manual-clock-rate-us" },
-	"If not zero, force this clock rate; if zero, try to autodetect it.",
-	0.0);
-
-static DoubleFlag noiseFloorFactor(
-    { "--noise-floor-factor" },
-    "Clock detection noise floor (min + (max-min)*factor).",
-    0.01);
-
-static DoubleFlag signalLevelFactor(
-    { "--signal-level-factor" },
-    "Clock detection signal level (min + (max-min)*factor).",
-    0.05);
-
-static const std::string BLOCK_ELEMENTS[] =
-{ " ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█" };
-
-/* 
-* Tries to guess the clock by finding the smallest common interval.
- * Returns nanoseconds.
- */
-nanoseconds_t Fluxmap::guessClock() const
+void AbstractDecoder::decodeToSectors(Track& track)
 {
-	if (manualClockRate != 0.0)
-		return manualClockRate * 1000.0;
+    Sector sector;
+    sector.physicalSide = track.physicalSide;
+    sector.physicalTrack = track.physicalTrack;
+    FluxmapReader fmr(*track.fluxmap);
 
-    uint32_t buckets[256] = {};
-    size_t cursor = 0;
-    FluxmapReader fr(*this);
-    while (cursor < bytes())
-    {
-        unsigned interval;
-        int opcode = fr.readPulse(interval);
-        if (opcode != 0x80)
-            break;
-        if (interval > 0xff)
-            continue;
-        buckets[interval]++;
-    }
-    
-    uint32_t max = *std::max_element(std::begin(buckets), std::end(buckets));
-    uint32_t min = *std::min_element(std::begin(buckets), std::end(buckets));
-    uint32_t noise_floor = min + (max-min)*noiseFloorFactor;
-    uint32_t signal_level = min + (max-min)*signalLevelFactor;
+    _track = &track;
+    _sector = &sector;
+    _fmr = &fmr;
 
-    /* Find a point solidly within the first pulse. */
-
-    int pulseindex = 0;
-    while (pulseindex < 256)
-    {
-        if (buckets[pulseindex] > signal_level)
-            break;
-        pulseindex++;
-    }
-    if (pulseindex == -1)
-        return 0;
-
-    /* Find the upper and lower bounds of the pulse. */
-
-    int peaklo = pulseindex;
-    while (peaklo > 0)
-    {
-        if (buckets[peaklo] < noise_floor)
-            break;
-        peaklo--;
-    }
-
-    int peakhi = pulseindex;
-    while (peakhi < 255)
-    {
-        if (buckets[peakhi] < noise_floor)
-            break;
-        peakhi++;
-    }
-
-    /* Find the total accumulated size of the pulse. */
-
-    uint32_t total_size = 0;
-    for (int i = peaklo; i < peakhi; i++)
-        total_size += buckets[i];
-
-    /* Now find the median. */
-
-    uint32_t count = 0;
-    int median = peaklo;
-    while (median < peakhi)
-    {
-        count += buckets[median];
-        if (count > (total_size/2))
-            break;
-        median++;
-    }
-
-    if (showClockHistogram)
-    {
-        std::cout << "Clock detection histogram:" << std::endl;
-        double blocks_per_count = 320.0/max;
-
-        auto show_noise_and_signal_levels = [=] {
-            /* Must be 12 chars in left margin */
-            std::cout << fmt::format("        0% >{:>{}}{:>{}}{:<{}}< 100%\n",
-                "|",
-                int((noise_floor*blocks_per_count)/8),
-                "|",
-                int((signal_level - noise_floor)*blocks_per_count/8),
-                "",
-                int((max - signal_level)*blocks_per_count/8));
-        };
-
-        show_noise_and_signal_levels();
-		bool skipping = true;
-        for (int i=0; i<256; i++)
-		{
-			uint32_t value = buckets[i];
-			if (value < noise_floor/2)
-			{
-				if (!skipping)
-					show_noise_and_signal_levels();
-				skipping = true;
-			}
-			else
-			{
-				skipping = false;
-
-				int bar = value * blocks_per_count;
-				int fullblocks = bar / 8;
-
-				std::string s;
-				for (int j=0; j<fullblocks; j++)
-					s += BLOCK_ELEMENTS[8];
-				s += BLOCK_ELEMENTS[bar & 7];
-
-                /* Must be 10 chars in left margin */
-				std::cout << fmt::format("{:5.2f}{:6} {}", (double)i * US_PER_TICK, value, s);
-				std::cout << std::endl;
-			}
-		}
-
-        std::cout << fmt::format("Noise floor:  {}", noise_floor) << std::endl;
-        std::cout << fmt::format("Signal level: {}", signal_level) << std::endl;
-        std::cout << fmt::format("Peak start:   {} ({:.2f} us)", peaklo, peaklo*US_PER_TICK) << std::endl;
-        std::cout << fmt::format("Peak end:     {} ({:.2f} us)", peakhi, peakhi*US_PER_TICK) << std::endl;
-        std::cout << fmt::format("Median:       {} ({:.2f} us)", median, median*US_PER_TICK) << std::endl;
-    }
-
-    /* 
-     * Okay, the median should now be a good candidate for the (or a) clock.
-     * How this maps onto the actual clock rate depends on the encoding.
-     */
-
-    return median * NS_PER_TICK;
-}
-
-/* Decodes a fluxmap into a nice aligned array of bits. */
-const RawBits Fluxmap::decodeToBits(nanoseconds_t clockPeriod) const
-{
-    int pulses = duration() / clockPeriod;
-    nanoseconds_t pulseAccuracy = pulseAccuracyFactor * clockPeriod;
-
-    auto bitmap = std::make_unique<std::vector<bool>>(pulses);
-    auto indices = std::make_unique<std::vector<size_t>>();
-
-    unsigned count = 0;
-    nanoseconds_t timestamp = 0;
-    FluxmapReader fr(*this);
+    beginTrack();
     for (;;)
     {
-        for (;;)
-        {
-            unsigned interval;
-            int opcode = fr.read(interval);
-            timestamp += interval * NS_PER_TICK;
-            if (opcode == -1)
-                goto abort;
-            else if (opcode == 0x80)
-                break;
-            else if (opcode == 0x81)
-                indices->push_back(count);
-        }
-
-        int clocks = (timestamp + clockPeriod/2) / clockPeriod;
-        nanoseconds_t expectedClock = clocks*clockPeriod;
-        if (abs(expectedClock - timestamp) > pulseAccuracy)
-            continue;
-
-        count += clocks;
-        if (count >= bitmap->size())
-            goto abort;
-        bitmap->at(count) = true;
-        timestamp = 0;
-    }
-abort:
-
-    RawBits rawbits(std::move(bitmap), std::move(indices));
-    return rawbits;
-}
-
-nanoseconds_t AbstractDecoder::guessClock(Fluxmap& fluxmap, unsigned physicalTrack) const
-{
-    return fluxmap.guessClock();
-}
-
-RawRecordVector AbstractSoftSectorDecoder::extractRecords(const RawBits& rawbits) const
-{
-    RawRecordVector records;
-    uint64_t fifo = 0;
-    size_t cursor = 0;
-    int matchStart = -1;
-
-    auto pushRecord = [&](size_t end)
-    {
-        if (matchStart == -1)
+        Fluxmap::Position recordStart = sector.position = fmr.tell();
+        sector.clock = 0;
+        sector.status = Sector::MISSING;
+        sector.data.clear();
+        sector.logicalSector = sector.logicalSide = sector.logicalTrack = 0;
+        RecordType r = advanceToNextRecord();
+        if (fmr.eof() || !sector.clock)
             return;
-
-        records.push_back(
-            std::unique_ptr<RawRecord>(
-                new RawRecord(
-                    matchStart,
-                    rawbits.begin() + matchStart,
-                    rawbits.begin() + end)
-            )
-        );
-    };
-
-    while (cursor < rawbits.size())
-    {
-        fifo = (fifo << 1) | rawbits[cursor++];
-        
-        int match = recordMatcher(fifo);
-        if (match > 0)
+        if ((r == UNKNOWN_RECORD) || (r == DATA_RECORD))
         {
-            pushRecord(cursor - match);
-            matchStart = cursor - match;
+            fmr.readNextMatchingOpcode(F_OP_PULSE);
+            continue;
         }
+
+        /* Read the sector record. */
+
+        recordStart = fmr.tell();
+        decodeSectorRecord();
+        pushRecord(recordStart, fmr.tell());
+        if (sector.status == Sector::DATA_MISSING)
+        {
+            /* The data is in a separate record. */
+
+            r = advanceToNextRecord();
+            if (r == DATA_RECORD)
+            {
+                recordStart = fmr.tell();
+                decodeDataRecord();
+                pushRecord(recordStart, fmr.tell());
+            }
+        }
+
+        if (sector.status != Sector::MISSING)
+            track.sectors.push_back(sector);
     }
-    pushRecord(cursor);
-    
-    return records;
 }
 
-RawRecordVector AbstractHardSectorDecoder::extractRecords(const RawBits& rawbits) const
+void AbstractDecoder::pushRecord(const Fluxmap::Position& start, const Fluxmap::Position& end)
 {
-    /* This is less easy than it looks.
-     *
-     * Hard-sectored disks contain one extra index hole, marking the top of the
-     * disk. This appears halfway in between two start-of-sector index holes. We
-     * need to find this and ignore it, otherwise we'll split that sector in two
-     * (and it won't work).
-     * 
-     * The routine here is pretty simple and requires this extra hole to have
-     * valid index holes either side of it --- it can't cope with the extra
-     * index hole being the first or last seen. Always give it slightly more
-     * than one revolution of data.
-     */
+    Fluxmap::Position here = _fmr->tell();
 
-    RawRecordVector records;
-    const auto& indices = rawbits.indices();
-    if (!indices.empty())
-    {
-        unsigned total = 0;
-        unsigned previous = 0;
-        for (unsigned index : indices)
-        {
-            total += index - previous;
-            previous = index;
-        }
-        total += rawbits.size() - previous;
+    RawRecord record;
+    record.physicalSide = _track->physicalSide;
+    record.physicalTrack = _track->physicalTrack;
+    record.clock = _sector->clock;
+    record.position = start;
 
-        unsigned sectors_must_be_bigger_than = (total / indices.size()) * 2/3;
-
-
-        previous = 0;
-        auto pushRecord = [&](size_t end)
-        {
-            if ((end - previous) < sectors_must_be_bigger_than)
-                return;
-
-            records.push_back(
-                std::unique_ptr<RawRecord>(
-                    new RawRecord(
-                        previous,
-                        rawbits.begin() + previous,
-                        rawbits.begin() + end)
-                )
-            );
-            previous = end;
-        };
-
-        for (unsigned index : indices)
-            pushRecord(index);
-        pushRecord(rawbits.size());
-    }
-
-    return records;
+    _fmr->seek(start);
+    record.data = toBytes(_fmr->readRawBits(end, _sector->clock));
+    _track->rawrecords.push_back(record);
+    _fmr->seek(here);
 }
-
-

lib/decoders/decoders.h

@@ -2,15 +2,22 @@
 #define DECODERS_H
 
 #include "bytes.h"
+#include "sector.h"
+#include "record.h"
+#include "decoders/fluxmapreader.h"
 
 class Sector;
 class Fluxmap;
+class FluxmapReader;
 class RawRecord;
 class RawBits;
+class Track;
 
 typedef std::vector<std::unique_ptr<RawRecord>> RawRecordVector;
 typedef std::vector<std::unique_ptr<Sector>> SectorVector;
 
+extern void setDecoderManualClockRate(double clockrate_us);
+
 extern Bytes decodeFmMfm(std::vector<bool>::const_iterator start,
     std::vector<bool>::const_iterator end);
 
@@ -22,28 +29,36 @@ class AbstractDecoder
 public:
     virtual ~AbstractDecoder() {}
 
-    virtual nanoseconds_t guessClock(Fluxmap& fluxmap, unsigned physicalTrack) const;
-    virtual RawRecordVector extractRecords(const RawBits& rawbits) const = 0;
-    virtual SectorVector decodeToSectors(const RawRecordVector& rawrecords,
-            unsigned physicalTrack) = 0;
-};
-
-class AbstractSoftSectorDecoder : public AbstractDecoder
-{
 public:
-    virtual ~AbstractSoftSectorDecoder() {}
+    enum RecordType
+    {
+        SECTOR_RECORD,
+        DATA_RECORD,
+        UNKNOWN_RECORD
+    };
 
-    RawRecordVector extractRecords(const RawBits& rawbits) const;
-
-    virtual int recordMatcher(uint64_t fifo) const = 0;
-};
-
-class AbstractHardSectorDecoder : public AbstractDecoder
-{
 public:
-    virtual ~AbstractHardSectorDecoder() {}
+    void decodeToSectors(Track& track);
+    void pushRecord(const Fluxmap::Position& start, const Fluxmap::Position& end);
 
-    RawRecordVector extractRecords(const RawBits& bits) const;
+    std::vector<bool> readRawBits(unsigned count)
+    { return _fmr->readRawBits(count, _sector->clock); }
+
+    Fluxmap::Position tell()
+    { return _fmr->tell(); } 
+
+    void seek(const Fluxmap::Position& pos)
+    { return _fmr->seek(pos); } 
+
+protected:
+    virtual void beginTrack() {};
+    virtual RecordType advanceToNextRecord() = 0;
+    virtual void decodeSectorRecord() = 0;
+    virtual void decodeDataRecord() {};
+
+    FluxmapReader* _fmr;
+    Track* _track;
+    Sector* _sector;
 };
 
 #endif

lib/decoders/fluxmapreader.cc

@@ -0,0 +1,284 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "decoders/fluxmapreader.h"
+#include "flags.h"
+#include "protocol.h"
+#include "fmt/format.h"
+#include <numeric>
+#include <math.h>
+#include <strings.h>
+
+FlagGroup fluxmapReaderFlags;
+
+DoubleFlag pulseDebounceThreshold(
+    { "--pulse-debounce-threshold" },
+    "Ignore pulses with intervals short than this, in fractions of a clock.",
+    0.30);
+
+static DoubleFlag clockDecodeThreshold(
+    { "--bit-error-threshold" },
+    "Amount of error to tolerate in pulse timing, in fractions of a clock.",
+    0.20);
+
+static DoubleFlag clockIntervalBias(
+    { "--clock-interval-bias" },
+    "Adjust intervals between pulses by this many clocks before decoding.",
+    -0.02);
+
+int FluxmapReader::readOpcode(unsigned& ticks)
+{
+    ticks = 0;
+
+    while (!eof())
+    {
+        uint8_t b = _bytes[_pos.bytes++];
+        if (b < 0x80)
+            ticks += b;
+        else
+        {
+            _pos.ticks += ticks;
+            return b;
+        }
+    }
+
+    _pos.ticks += ticks;
+    return -1;
+}
+
+unsigned FluxmapReader::readNextMatchingOpcode(uint8_t opcode)
+{
+    unsigned ticks = 0;
+
+    for (;;)
+    {
+        unsigned thisTicks;
+        int op = readOpcode(thisTicks);
+        ticks += thisTicks;
+        if (op == -1)
+            return 0;
+        if (op == opcode)
+            return ticks;
+    }
+}
+
+unsigned FluxmapReader::readInterval(nanoseconds_t clock)
+{
+    unsigned thresholdTicks = (clock * pulseDebounceThreshold) / NS_PER_TICK;
+    unsigned ticks = 0;
+
+    while (ticks < thresholdTicks)
+    {
+        unsigned thisTicks = readNextMatchingOpcode(F_OP_PULSE);
+        if (!thisTicks)
+            break;
+        ticks += thisTicks;
+    }
+    return ticks;
+}
+
+static int findLowestSetBit(uint64_t value)
+{
+    if (!value)
+        return 0;
+    int bit = 1;
+    while (!(value & 1))
+    {
+        value >>= 1;
+        bit++;
+    }
+    return bit;
+}
+
+FluxPattern::FluxPattern(unsigned bits, uint64_t pattern):
+    _bits(bits)
+{
+    const uint64_t TOPBIT = 1ULL << 63;
+
+    assert(pattern != 0);
+
+    unsigned lowbit = findLowestSetBit(pattern)-1;
+
+    pattern <<= 64 - bits;
+    _highzeroes = 0;
+    while (!(pattern & TOPBIT))
+    {
+        pattern <<= 1;
+        _highzeroes++;
+    }
+
+    _length = 0;
+    while (pattern != TOPBIT)
+    {
+        unsigned interval = 0;
+        do
+        {
+            pattern <<= 1;
+            interval++;
+        }
+        while (!(pattern & TOPBIT));
+        _intervals.push_back(interval);
+        _length += interval;
+    }
+
+    if (lowbit)
+    {
+        _lowzero = true;
+        /* Note that length does *not* include this interval. */
+        _intervals.push_back(lowbit + 1);
+    }
+}
+
+bool FluxPattern::matches(const unsigned* end, FluxMatch& match) const
+{
+    const unsigned* start = end - _intervals.size();
+    unsigned candidatelength = std::accumulate(start, end - _lowzero, 0);
+    if (!candidatelength)
+        return false;
+    match.clock = (double)candidatelength / (double)_length;
+
+    unsigned exactIntervals = _intervals.size() - _lowzero;
+    for (unsigned i=0; i<exactIntervals; i++)
+    {
+        double ii = match.clock * (double)_intervals[i];
+        double ci = (double)start[i];
+        double error = fabs((ii - ci) / match.clock);
+        if (error > clockDecodeThreshold)
+            return false;
+    }
+
+    if (_lowzero)
+    {
+        double ii = match.clock * (double)_intervals[exactIntervals];
+        double ci = (double)start[exactIntervals];
+        double error = (ii - ci) / match.clock;
+        if (error > clockDecodeThreshold)
+            return false;
+    }
+
+    match.matcher = this;
+    match.intervals = _intervals.size();
+    match.zeroes = _highzeroes;
+    return true;
+}
+
+FluxMatchers::FluxMatchers(const std::initializer_list<const FluxMatcher*> matchers):
+    _matchers(matchers)
+{
+    _intervals = 0;
+    for (const auto* matcher : matchers)
+        _intervals = std::max(_intervals, matcher->intervals());
+}
+
+bool FluxMatchers::matches(const unsigned* intervals, FluxMatch& match) const
+{
+    for (const auto* matcher : _matchers)
+    {
+        if (matcher->matches(intervals, match))
+            return true;
+    }
+    return false;
+}
+
+void FluxmapReader::seek(nanoseconds_t ns)
+{
+    unsigned ticks = ns / NS_PER_TICK;
+    if (ticks < _pos.ticks)
+    {
+        _pos.ticks = 0;
+        _pos.bytes = 0;
+    }
+
+    while (!eof() && (_pos.ticks < ticks))
+    {
+        unsigned t;
+        readOpcode(t);
+    }
+    _pos.zeroes = 0;
+}
+
+nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern)
+{
+    const FluxMatcher* unused;
+    return seekToPattern(pattern, unused);
+}
+
+nanoseconds_t FluxmapReader::seekToPattern(const FluxMatcher& pattern, const FluxMatcher*& matching)
+{
+    unsigned intervalCount = pattern.intervals();
+    unsigned candidates[intervalCount+1];
+    Fluxmap::Position positions[intervalCount+1];
+
+    for (unsigned i=0; i<=intervalCount; i++)
+    {
+        positions[i] = tell();
+        candidates[i] = 0;
+    }
+
+    while (!eof())
+    {
+        FluxMatch match;
+        if (pattern.matches(&candidates[intervalCount+1], match))
+        {
+            seek(positions[intervalCount-match.intervals]);
+            _pos.zeroes = match.zeroes;
+            matching = match.matcher;
+            return match.clock * NS_PER_TICK;
+        }
+
+        for (unsigned i=0; i<intervalCount; i++)
+        {
+            positions[i] = positions[i+1];
+            candidates[i] = candidates[i+1];
+        }
+        candidates[intervalCount] = readNextMatchingOpcode(F_OP_PULSE);
+        positions[intervalCount] = tell();
+
+    }
+
+    matching = NULL;
+    return 0;
+}
+
+void FluxmapReader::seekToIndexMark()
+{
+    readNextMatchingOpcode(F_OP_INDEX);
+    _pos.zeroes = 0;
+}
+
+bool FluxmapReader::readRawBit(nanoseconds_t clockPeriod)
+{
+    assert(clockPeriod != 0);
+
+    if (_pos.zeroes)
+    {
+        _pos.zeroes--;
+        return false;
+    }
+
+    nanoseconds_t interval = readInterval(clockPeriod)*NS_PER_TICK;
+    double clocks = (double)interval / clockPeriod + clockIntervalBias;
+
+    if (clocks < 1.0)
+        clocks = 1.0;
+    _pos.zeroes = (int)round(clocks) - 1;
+    return true;
+}
+
+std::vector<bool> FluxmapReader::readRawBits(unsigned count, nanoseconds_t clockPeriod)
+{
+    std::vector<bool> result;
+    while (!eof() && count--)
+    {
+        bool b = readRawBit(clockPeriod);
+        result.push_back(b);
+    }
+    return result;
+}
+
+std::vector<bool> FluxmapReader::readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod)
+{
+    std::vector<bool> result;
+    while (!eof() && (_pos.bytes < until.bytes))
+        result.push_back(readRawBit(clockPeriod));
+    return result;
+}

lib/decoders/fluxmapreader.h

@@ -0,0 +1,121 @@
+#ifndef FLUXMAPREADER_H
+#define FLUXMAPREADER_H
+
+#include "fluxmap.h"
+#include "protocol.h"
+#include "flags.h"
+
+extern FlagGroup fluxmapReaderFlags;
+
+class FluxMatcher;
+
+struct FluxMatch
+{
+    const FluxMatcher* matcher;
+    unsigned intervals;
+    double clock;
+    unsigned zeroes;
+};
+
+class FluxMatcher
+{
+public:
+    virtual ~FluxMatcher() {}
+
+    /* Returns the number of intervals matched */
+    virtual bool matches(const unsigned* intervals, FluxMatch& match) const = 0;
+    virtual unsigned intervals() const = 0;
+};
+
+class FluxPattern : public FluxMatcher
+{
+public:
+    FluxPattern(unsigned bits, uint64_t patterns);
+
+    bool matches(const unsigned* intervals, FluxMatch& match) const override;
+
+    unsigned intervals() const override
+    { return _intervals.size(); }
+
+private:
+    std::vector<unsigned> _intervals;
+    unsigned _length;
+    unsigned _bits;
+    unsigned _highzeroes;
+    bool _lowzero = false;
+
+public:
+    friend void test_patternconstruction();
+    friend void test_patternmatching();
+};
+
+class FluxMatchers : public FluxMatcher
+{
+public:
+    FluxMatchers(const std::initializer_list<const FluxMatcher*> matchers);
+
+    bool matches(const unsigned* intervals, FluxMatch& match) const override;
+
+    unsigned intervals() const override
+    { return _intervals; }
+
+private:
+    unsigned _intervals;
+    std::vector<const FluxMatcher*> _matchers;
+};
+
+class FluxmapReader
+{
+public:
+    FluxmapReader(const Fluxmap& fluxmap):
+        _fluxmap(fluxmap),
+        _bytes(fluxmap.ptr()),
+        _size(fluxmap.bytes())
+    {
+        rewind();
+    }
+
+    FluxmapReader(const Fluxmap&& fluxmap) = delete;
+
+    void rewind()
+    {
+        _pos.bytes = 0;
+        _pos.ticks = 0;
+        _pos.zeroes = 0;
+    }
+
+    bool eof() const
+    { return _pos.bytes == _size; }
+
+    Fluxmap::Position tell() const
+    { return _pos; }
+
+    /* Important! You can only reliably seek to 1 bits. */
+    void seek(const Fluxmap::Position& pos)
+    {
+        _pos = pos;
+    }
+
+    int readOpcode(unsigned& ticks);
+    unsigned readNextMatchingOpcode(uint8_t opcode);
+    unsigned readInterval(nanoseconds_t clock); /* with debounce support */
+
+    /* Important! You can only reliably seek to 1 bits. */
+    void seek(nanoseconds_t ns);
+
+    void seekToIndexMark();
+    nanoseconds_t seekToPattern(const FluxMatcher& pattern);
+    nanoseconds_t seekToPattern(const FluxMatcher& pattern, const FluxMatcher*& matching);
+
+    bool readRawBit(nanoseconds_t clockPeriod);
+    std::vector<bool> readRawBits(unsigned count, nanoseconds_t clockPeriod);
+    std::vector<bool> readRawBits(const Fluxmap::Position& until, nanoseconds_t clockPeriod);
+
+private:
+    const Fluxmap& _fluxmap;
+    const uint8_t* _bytes;
+    const size_t _size;
+    Fluxmap::Position _pos;
+};
+
+#endif

lib/decoders/fluxreader/hardwarefluxreader.cc.orig

@@ -9,6 +9,13 @@ static IntFlag revolutions(
     "read this many revolutions of the disk",
     1);
 
+static bool high_density = false;
+
+void setHardwareFluxReaderDensity(bool high_density)
+{
+	::high_density = high_density;
+}
+
 class HardwareFluxReader : public FluxReader
 {
 public:
@@ -24,9 +31,12 @@ public:
 public:
     std::unique_ptr<Fluxmap> readFlux(int track, int side)
     {
-        usbSetDrive(_drive);
+        usbSetDrive(_drive, high_density);
         usbSeek(track);
-        return usbRead(side, revolutions);
+        Bytes crunched = usbRead(side, revolutions);
+        auto fluxmap = std::make_unique<Fluxmap>();
+        fluxmap->appendBytes(crunched.uncrunch());
+        return fluxmap;
     }
 
     void recalibrate()

lib/decoders/fmmfm.cc

@@ -1,5 +1,5 @@
 #include "globals.h"
-#include "decoders.h"
+#include "decoders/decoders.h"
 
 Bytes decodeFmMfm(
         std::vector<bool>::const_iterator ii, std::vector<bool>::const_iterator end)
@@ -26,7 +26,7 @@ Bytes decodeFmMfm(
     ByteWriter bw(bytes);
 
     int bitcount = 0;
-    uint8_t fifo;
+    uint8_t fifo = 0;
 
     while (ii != end)
     {

lib/encoders/encoders.cc

@@ -12,6 +12,7 @@ Fluxmap& Fluxmap::appendBits(const std::vector<bool>& bits, nanoseconds_t clock)
 		{
 			unsigned delta = (now - duration()) / NS_PER_TICK;
             appendInterval(delta);
+			appendPulse();
 		}
 	}
 

lib/encoders/encoders.h

@@ -0,0 +1,18 @@
+#ifndef ENCODERS_H
+#define ENCODERS_H
+
+class Fluxmap;
+class SectorSet;
+
+class AbstractEncoder
+{
+public:
+    virtual ~AbstractEncoder() {}
+
+public:
+    virtual std::unique_ptr<Fluxmap> encode(
+        int physicalTrack, int physicalSide, const SectorSet& allSectors) = 0;
+};
+
+#endif
+

lib/f85/decoder.cc

@@ -1,108 +0,0 @@
-#include "globals.h"
-#include "fluxmap.h"
-#include "protocol.h"
-#include "record.h"
-#include "decoders.h"
-#include "sector.h"
-#include "f85.h"
-#include "crc.h"
-#include "bytes.h"
-#include "fmt/format.h"
-#include <string.h>
-#include <algorithm>
-
-static int decode_data_gcr(uint8_t gcr)
-{
-    switch (gcr)
-    {
-		#define GCR_ENTRY(gcr, data) \
-			case gcr: return data;
-		#include "data_gcr.h"
-		#undef GCR_ENTRY
-    }
-    return -1;
-};
-
-static Bytes decode(const std::vector<bool>& bits)
-{
-    Bytes output;
-    ByteWriter bw(output);
-    BitWriter bitw(bw);
-
-    auto ii = bits.begin();
-    while (ii != bits.end())
-    {
-        uint8_t inputfifo = 0;
-        for (size_t i=0; i<5; i++)
-        {
-            if (ii == bits.end())
-                break;
-            inputfifo = (inputfifo<<1) | *ii++;
-        }
-
-        bitw.push(decode_data_gcr(inputfifo), 4);
-    }
-    bitw.flush();
-
-    return output;
-}
-
-SectorVector DurangoF85Decoder::decodeToSectors(const RawRecordVector& rawRecords, unsigned)
-{
-    std::vector<std::unique_ptr<Sector>> sectors;
-    unsigned nextSector;
-    unsigned nextTrack;
-    bool headerIsValid = false;
-
-    for (auto& rawrecord : rawRecords)
-    {
-        const auto& rawdata = rawrecord->data;
-        const auto& bytes = decode(rawdata);
-
-        if (bytes.size() < 4)
-            continue;
-
-        switch (bytes[0])
-        {
-            case 0xce: /* sector record */
-            {
-                headerIsValid = false;
-                nextSector = bytes[3];
-                nextTrack = bytes[1];
-
-                uint16_t wantChecksum = bytes.reader().seek(5).read_be16();
-                uint16_t gotChecksum = crc16(CCITT_POLY, 0xef21, bytes.slice(1, 4));
-                headerIsValid = (wantChecksum == gotChecksum);
-                break;
-            }
-
-            case 0xcb: /* data record */
-            {
-                if (!headerIsValid)
-                    break;
-                if (bytes.size() < (F85_SECTOR_LENGTH + 3))
-                    continue;
-
-                const auto& payload = bytes.slice(1, F85_SECTOR_LENGTH);
-                uint16_t wantChecksum = bytes.reader().seek(F85_SECTOR_LENGTH+1).read_be16();
-                uint16_t gotChecksum = crc16(CCITT_POLY, 0xbf84, payload);
-
-                int status = (wantChecksum == gotChecksum) ? Sector::OK : Sector::BAD_CHECKSUM;
-                auto sector = std::unique_ptr<Sector>(
-					new Sector(status, nextTrack, 0, nextSector, payload));
-                sectors.push_back(std::move(sector));
-                break;
-            }
-        }
-	}
-
-	return sectors;
-}
-
-int DurangoF85Decoder::recordMatcher(uint64_t fifo) const
-{
-    uint32_t masked = fifo & 0xffff;
-    if (masked == F85_RECORD_SEPARATOR)
-		return 6;
-    return 0;
-}

lib/f85/f85.h

@@ -1,20 +0,0 @@
-#ifndef F85_H
-#define F85_H
-
-#define F85_RECORD_SEPARATOR 0xfffc
-#define F85_SECTOR_LENGTH    512
-
-class Sector;
-class Fluxmap;
-
-class DurangoF85Decoder : public AbstractSoftSectorDecoder
-{
-public:
-    virtual ~DurangoF85Decoder() {}
-
-    SectorVector decodeToSectors(
-        const RawRecordVector& rawRecords, unsigned physicalTrack);
-    int recordMatcher(uint64_t fifo) const;
-};
-
-#endif

lib/flags.cc

@@ -1,25 +1,71 @@
 #include "globals.h"
 #include "flags.h"
 
+static FlagGroup* currentFlagGroup;
 static std::vector<Flag*> all_flags;
 static std::map<const std::string, Flag*> flags_by_name;
 
-Flag::Flag(const std::vector<std::string>& names, const std::string helptext):
-    _names(names),
-    _helptext(helptext)
-{
-    for (auto& name : names)
-    {
-        if (flags_by_name.find(name) != flags_by_name.end())
-            Error() << "two flags use the name '" << name << "'";
-        flags_by_name[name] = this;
-    }
+static void doHelp();
 
-    all_flags.push_back(this);
+static FlagGroup helpGroup;
+static ActionFlag helpFlag = ActionFlag(
+    { "--help", "-h" },
+    "Shows the help.",
+    doHelp);
+
+FlagGroup::FlagGroup(const std::initializer_list<FlagGroup*> groups):
+    _groups(groups.begin(), groups.end())
+{
+    currentFlagGroup = this;
 }
 
-void Flag::parseFlags(int argc, const char* argv[])
+FlagGroup::FlagGroup()
 {
+    currentFlagGroup = this;
+}
+
+void FlagGroup::addFlag(Flag* flag)
+{
+    _flags.push_back(flag);
+}
+
+void FlagGroup::parseFlags(int argc, const char* argv[])
+{
+    if (_initialised)
+        throw std::runtime_error("called parseFlags() twice");
+
+    /* Recursively accumulate a list of all flags. */
+
+    all_flags.clear();
+    flags_by_name.clear();
+    std::function<void(FlagGroup*)> recurse;
+    recurse = [&](FlagGroup* group)
+    {
+        if (group->_initialised)
+            return;
+        
+        for (FlagGroup* subgroup : group->_groups)
+            recurse(subgroup);
+
+        for (Flag* flag : group->_flags)
+        {
+            for (const auto& name : flag->names())
+            {
+                if (flags_by_name.find(name) != flags_by_name.end())
+                    Error() << "two flags use the name '" << name << "'";
+                flags_by_name[name] = flag;
+            }
+
+            all_flags.push_back(flag);
+        }
+
+        group->_initialised = true;
+    };
+    recurse(this);
+    recurse(&helpGroup);
+
+    /* Now actually parse them. */
+
     int index = 1;
     while (index < argc)
     {
@@ -76,15 +122,28 @@ void Flag::parseFlags(int argc, const char* argv[])
         if (usesthat && flag->second->hasArgument())
             index++;
     }
-        
+}
+
+void FlagGroup::checkInitialised() const
+{
+    if (!_initialised)
+        throw std::runtime_error("Attempt to access uninitialised flag");
+}
+
+Flag::Flag(const std::vector<std::string>& names, const std::string helptext):
+    _group(*currentFlagGroup),
+    _names(names),
+    _helptext(helptext)
+{
+    _group.addFlag(this);
 }
 
 void BoolFlag::set(const std::string& value)
 {
 	if ((value == "true") || (value == "y"))
-		this->value = true;
+		_value = true;
 	else if ((value == "false") || (value == "n"))
-		this->value = false;
+		_value = false;
 	else
 		Error() << "can't parse '" << value << "'; try 'true' or 'false'";
 }
@@ -110,8 +169,3 @@ static void doHelp()
     }
     exit(0);
 }
-
-static ActionFlag helpFlag = ActionFlag(
-    { "--help", "-h" },
-    "Shows the help.",
-    doHelp);

lib/flags.h

@@ -2,15 +2,36 @@
 #define FLAGS_H
 
 class DataSpec;
+class Flag;
+
+class FlagGroup
+{
+private:
+    FlagGroup(const FlagGroup& group);
+public:
+    FlagGroup(const std::initializer_list<FlagGroup*> groups);
+    FlagGroup();
+
+public:
+    void parseFlags(int argc, const char* argv[]);
+    void addFlag(Flag* flag);
+    void checkInitialised() const;
+
+private:
+    bool _initialised = false;
+    const std::vector<FlagGroup*> _groups;
+    std::vector<Flag*> _flags;
+};
 
 class Flag
 {
 public:
-    static void parseFlags(int argc, const char* argv[]);
-
     Flag(const std::vector<std::string>& names, const std::string helptext);
     virtual ~Flag() {};
 
+    void checkInitialised() const
+    { _group.checkInitialised(); }
+
     const std::string& name() const { return _names[0]; }
     const std::vector<std::string> names() const { return _names; }
     const std::string& helptext() const { return _helptext; }
@@ -20,6 +41,7 @@ public:
     virtual void set(const std::string& value) = 0;
 
 private:
+    FlagGroup& _group;
     const std::vector<std::string> _names;
     const std::string _helptext;
 };
@@ -48,7 +70,8 @@ public:
         Flag(names, helptext)
     {}
 
-    operator bool() const { return _value; }
+    operator bool() const
+    { checkInitialised(); return _value; }
 
     bool hasArgument() const { return false; }
     const std::string defaultValueAsString() const { return "false"; }
@@ -65,16 +88,26 @@ public:
     ValueFlag(const std::vector<std::string>& names, const std::string helptext,
             const T defaultValue):
         Flag(names, helptext),
-        defaultValue(defaultValue),
-        value(defaultValue)
+        _defaultValue(defaultValue),
+        _value(defaultValue)
     {}
 
-    operator T() const { return value; }
+    const T& get() const
+    { checkInitialised(); return _value; }
+
+    operator const T& () const 
+    { return get(); }
+
+    void setDefaultValue(T value)
+    {
+        _value = _defaultValue = value;
+    }
 
     bool hasArgument() const { return true; }
 
-    T defaultValue;
-    T value;
+protected:
+    T _defaultValue;
+    T _value;
 };
 
 class StringFlag : public ValueFlag<std::string>
@@ -85,8 +118,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return defaultValue; }
-    void set(const std::string& value) { this->value = value; }
+    const std::string defaultValueAsString() const { return _defaultValue; }
+    void set(const std::string& value) { _value = value; }
 };
 
 class IntFlag : public ValueFlag<int>
@@ -97,8 +130,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return std::to_string(defaultValue); }
-    void set(const std::string& value) { this->value = std::stoi(value); }
+    const std::string defaultValueAsString() const { return std::to_string(_defaultValue); }
+    void set(const std::string& value) { _value = std::stoi(value); }
 };
 
 class DoubleFlag : public ValueFlag<double>
@@ -109,8 +142,8 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return std::to_string(defaultValue); }
-    void set(const std::string& value) { this->value = std::stod(value); }
+    const std::string defaultValueAsString() const { return std::to_string(_defaultValue); }
+    void set(const std::string& value) { _value = std::stod(value); }
 };
 
 class BoolFlag : public ValueFlag<double>
@@ -121,7 +154,7 @@ public:
         ValueFlag(names, helptext, defaultValue)
     {}
 
-    const std::string defaultValueAsString() const { return defaultValue ? "true" : "false"; }
+    const std::string defaultValueAsString() const { return _defaultValue ? "true" : "false"; }
     void set(const std::string& value);
 };
 

lib/fluxmap.cc

@@ -32,6 +32,11 @@ Fluxmap& Fluxmap::appendInterval(uint32_t ticks)
         ticks -= 0x7f;
     }
     appendByte((uint8_t)ticks);
+    return *this;
+}
+
+Fluxmap& Fluxmap::appendPulse()
+{
     appendByte(0x80);
     return *this;
 }
@@ -72,33 +77,3 @@ void Fluxmap::precompensate(int threshold_ticks, int amount_ticks)
         }
     }
 }
-
-int FluxmapReader::read(unsigned& ticks)
-{
-    ticks = 0;
-
-    while (_cursor < _size)
-    {
-        uint8_t b = _bytes[_cursor++];
-        if (b < 0x80)
-            ticks += b;
-        else
-            return b;
-    }
-
-    return -1;
-}
-
-int FluxmapReader::readPulse(unsigned& ticks)
-{
-    ticks = 0;
-
-    for (;;)
-    {
-        unsigned thisTicks;
-        int opcode = read(thisTicks);
-        ticks += thisTicks;
-        if ((opcode == -1) || (opcode == 0x80))
-            return opcode;
-    }
-}