Add support for reading Kryoflux stream files.

.hgignore

@@ -1,2 +1,5 @@
 .obj
+streams
+.*\.flux
+.*\.img
 

README.md

@@ -46,6 +46,13 @@ to be spinning at the same speed.
 which means there's only 166ms of data on one per track rather than 200ms;
 if you try to write a 3.5" format disk onto one it probably won't work.
 
+**Q.** Is this like KryoFlux? Do you support KryoFlux stream files?
+
+**A.** It's very like KryoFlux, although much simpler. Yes, FluxEngine can
+read from KryoFlux stream files (but not write to them yet; nobody's asked).
+FluxEngine doesn't capture all the data that KryoFlux does, like index
+markers.
+
 **Q.** That's awesome! What formats does it support?
 
 **A.** I'm glad you asked the question. Not a lot, currently.
@@ -66,7 +73,8 @@ Currently, not a lot.
 
   - [Acorn DFS disks](doc/acorn-dfs.md): read only (likewise)
 
-  - [Brother 240kB word processor disks](doc/brother.md); read and write
+  - [Brother 120kB and 240kB word processor disks](doc/brother.md); read and
+    write
 
 ...aaaand that's it. If you want more, please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new); I need samples
@@ -279,7 +287,12 @@ fe-readibm -s fakedisk.flux:t=0-79:s=0
     `:t=0-3` and `:t=0,1,2,3` are equivalent.
 
   - When specifying a range, you can also specify the step. For example,
-    `:t=0-79x2` would be used when accessing a 40-track disk with double stepping.
+    `:t=0-79x2` would be used when accessing a 40-track disk with double
+    stepping.
+
+  - To read from a set of KryoFlux stream files, specify the path to the
+    directory containing the files _with a trailing slash_; so
+    `some/files/:t=0-10`.
 
 Source and destination specifiers work entirely in *physical units*.
 FluxEngine is intended to be connected to an 80 (or 82) track double sided
@@ -400,6 +413,10 @@ Useful links:
     the technical data sheet for a representative drive. Lots of useful
     timing numbers here.
 
+  - [KryoFlux stream file
+    documentation](https://www.kryoflux.com/download/kryoflux_stream_protocol_rev1.1.pdf):
+    the format of KryoFlux stream files (partially supported by FluxEngine)
+
 Who?
 ----
 

doc/brother.md

@@ -5,8 +5,10 @@ Brother word processor disks are weird, using custom tooling and chipsets.
 They are completely not PC compatible in every possible way other than the
 size.
 
-Different word processors use different disk formats --- the only one
+Different word processors use different disk formats --- the only ones
-supported by FluxEngine is the 240kB 3.5" format.
+supported by FluxEngine are the 120kB and 240kB 3.5" formats. The default
+options are for the 240kB format. For the 120kB format, which is 40 track, do
+`fe-readbrother -s :t=1-79x2`.
 
 Apparently about 20% of Brother word processors have alignment issues which
 means that the disks can't be read by FluxEngine (because the tracks on the
@@ -52,7 +54,9 @@ Low level format
 ----------------
 
 The drive is a single-sided 3.5" drive spinning at not 300 rpm (I don't know
-the precise speed yet but FluxEngine doesn't care). The disks have 78 tracks.
+the precise speed yet but FluxEngine doesn't care). The 240kB disks have 78
+tracks and the 120kB disks have 39.
+
 The Brother drive alignment is kinda variable; when you put the disk in the
 drive it seeks all the way to physical track 0 and then starts searching for
 something which looks like data. My machine likes to put logical track 0 on

lib/brother/parser.cc

@@ -25,7 +25,7 @@ std::vector<std::unique_ptr<Sector>> BrotherRecordParser::parseRecordsToSectors(
 					goto garbage;
 				nextTrack = data[1];
 				nextSector = data[2];
-				hasHeader = true;
+				hasHeader = (nextTrack != 0xff) && (nextSector != 0xff);
                 break;
 
             case BROTHER_DATA_RECORD & 0xff:

lib/decoders/decoders.cc

@@ -8,7 +8,7 @@
 static IntFlag clockDetectionNoiseFloor(
     { "--clock-detection-noise-floor" },
     "Noise floor used for clock detection in flux.",
-    50);
+    200);
 
 static DoubleFlag clockDecodeThreshold(
     { "--clock-decode-threshold" },
@@ -29,10 +29,10 @@ nanoseconds_t Fluxmap::guessClock() const
     for (uint8_t interval : _intervals)
         buckets[interval]++;
     
-    int peaklo = 0;
+    int peaklo = 1;
     while (peaklo < 256)
     {
-        if (buckets[peaklo] > 100)
+        if (buckets[peaklo] > (uint32_t)(clockDetectionNoiseFloor*2))
             break;
         peaklo++;
     }

lib/fluxmap.cc

@@ -9,8 +9,6 @@ Fluxmap& Fluxmap::appendIntervals(const std::vector<uint8_t>& intervals)
 
 Fluxmap& Fluxmap::appendIntervals(const uint8_t* ptr, size_t len)
 {
-    _intervals.reserve(_intervals.size() + len);
-
     while (len--)
     {
         uint8_t interval = *ptr++;

lib/fluxmap.h

@@ -22,6 +22,11 @@ public:
     Fluxmap& appendIntervals(const std::vector<uint8_t>& intervals);
     Fluxmap& appendIntervals(const uint8_t* ptr, size_t len);
 
+    Fluxmap& appendInterval(uint8_t interval)
+    {
+        return appendIntervals(&interval, 1);
+    }
+
     nanoseconds_t guessClock() const;
 	std::vector<bool> decodeToBits(nanoseconds_t clock_period) const;
 

lib/fluxreader/fluxreader.cc

@@ -18,6 +18,8 @@ std::unique_ptr<FluxReader> FluxReader::create(const DataSpec& spec)
         return createHardwareFluxReader(spec.drive);
     else if (ends_with(filename, ".flux"))
         return createSqliteFluxReader(filename);
+    else if (ends_with(filename, "/"))
+        return createStreamFluxReader(filename);
 
     Error() << "unrecognised flux filename extension";
     return std::unique_ptr<FluxReader>();

lib/fluxreader/fluxreader.h

@@ -12,6 +12,7 @@ public:
 private:
     static std::unique_ptr<FluxReader> createSqliteFluxReader(const std::string& filename);
     static std::unique_ptr<FluxReader> createHardwareFluxReader(unsigned drive);
+    static std::unique_ptr<FluxReader> createStreamFluxReader(const std::string& path);
 
 public:
     static std::unique_ptr<FluxReader> create(const DataSpec& spec);

lib/fluxreader/streamfluxreader.cc

@@ -0,0 +1,33 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "stream.h"
+#include "fluxreader.h"
+
+class StreamFluxReader : public FluxReader
+{
+public:
+    StreamFluxReader(const std::string& path):
+        _path(path)
+    {
+    }
+
+    ~StreamFluxReader()
+    {
+    }
+
+public:
+    std::unique_ptr<Fluxmap> readFlux(int track, int side)
+    {
+        return readStream(_path, track, side);
+    }
+
+    void recalibrate() {}
+
+private:
+    const std::string& _path;
+};
+
+std::unique_ptr<FluxReader> FluxReader::createStreamFluxReader(const std::string& path)
+{
+    return std::unique_ptr<FluxReader>(new StreamFluxReader(path));
+}

lib/reader.cc

@@ -55,7 +55,7 @@ std::unique_ptr<Fluxmap> Track::read()
 	std::cout << fmt::format(
 		"{0} ms in {1} bytes", int(fluxmap->duration()/1e6), fluxmap->bytes()) << std::endl;
 	if (outdb)
-		sqlWriteFlux(outdb, track, track, *fluxmap);
+		sqlWriteFlux(outdb, track, side, *fluxmap);
 	return fluxmap;
 }
 

lib/stream/stream.cc

@@ -0,0 +1,101 @@
+#include "globals.h"
+#include "fluxmap.h"
+#include "stream.h"
+#include "protocol.h"
+#include "fmt/format.h"
+#include <fstream>
+
+#define SCLK_HZ 24027428.57142857
+#define TICKS_PER_SCLK (TICK_FREQUENCY / SCLK_HZ)
+
+std::unique_ptr<Fluxmap> readStream(const std::string& path, unsigned track, unsigned side)
+{
+    auto filename = fmt::format("{}track{:02}.{}.raw", path, track, side);
+    std::ifstream f(filename, std::ios::in | std::ios::binary);
+    if (!f.is_open())
+		Error() << fmt::format("cannot open input file '{}'", filename);
+
+    std::unique_ptr<Fluxmap> fluxmap(new Fluxmap);
+    auto writeFlux = [&](uint32_t sclk)
+    {
+        int ticks = (double)sclk * TICKS_PER_SCLK;
+        while (ticks >= 0x100)
+        {
+            fluxmap->appendInterval(0);
+            ticks -= 0x100;
+        }
+        fluxmap->appendInterval((uint8_t)ticks);
+    };
+
+    for (;;)
+    {
+        int b = f.get(); /* returns -1 or UNSIGNED char */
+        if (b == -1)
+            break;
+        uint64_t here = f.tellg();
+
+        switch (b)
+        {
+            case 0x0d: /* OOB block */
+            {
+                int blocktype = f.get();
+                int blocklen = f.get() | (f.get()<<8);
+                if (f.fail() || f.eof())
+                    goto finished;
+
+                f.seekg(here + blocklen, std::ios_base::beg);
+                break;
+            }
+
+            default:
+            {
+                if ((b >= 0x00) && (b <= 0x07))
+                {
+                    /* Flux2: double byte value */
+                    b = (b<<8) | f.get();
+                    writeFlux(b);
+                }
+                else if (b == 0x08)
+                {
+                    /* Nop1: do nothing */
+                }
+                else if (b == 0x09)
+                {
+                    /* Nop2: skip one byte */
+                    f.seekg(1, std::ios_base::cur);
+                }
+                else if (b == 0x0a)
+                {
+                    /* Nop3: skip two bytes */
+                    f.seekg(2, std::ios_base::cur);
+                }
+                else if (b == 0x0b)
+                {   /* Ovl16: the next block is 0x10000 sclks longer than normal.
+                     * FluxEngine can't handle long transitions, and implementing
+                     * this is complicated, so we just bodge it.
+                     */
+                    writeFlux(0x10000);
+                }
+                else if (b == 0x0c)
+                {
+                    /* Flux3: triple byte value */
+                    int ticks = f.get() | (f.get()<<8);
+                    writeFlux(ticks);
+                }
+                else if ((b >= 0x0e) && (b <= 0xff))
+                {
+                    /* Flux1: single byte value */
+                    writeFlux(b);
+                }
+                else
+                    Error() << fmt::format(
+                        "unknown stream block byte 0x{:02x} at 0x{:08x}", b, here);
+            }
+        }
+    }
+
+finished:
+    if (!f.eof())
+        Error() << fmt::format("I/O error reading '{}'", filename);
+    return fluxmap;
+}

lib/stream/stream.h

@@ -0,0 +1,6 @@
+#ifndef STREAM_H
+#define STREAM_H
+
+extern std::unique_ptr<Fluxmap> readStream(const std::string& path, unsigned track, unsigned side);
+
+#endif

meson.build

@@ -36,14 +36,22 @@ sqllib = shared_library('sqllib',
 	dependencies: [sqlite]
 )
 
+streamlib = shared_library('streamlib',
+    [ 'lib/stream/stream.cc', ],
+    include_directories: [feinc, fmtinc],
+    link_with: [felib, fmtlib]
+)
+streaminc = include_directories('lib/stream')
+
 fluxreaderlib = shared_library('fluxreaderlib',
     [
         'lib/fluxreader/fluxreader.cc',
         'lib/fluxreader/sqlitefluxreader.cc',
         'lib/fluxreader/hardwarefluxreader.cc',
+        'lib/fluxreader/streamfluxreader.cc',
     ],
-    include_directories: [feinc, fmtinc],
+    include_directories: [feinc, fmtinc, streaminc],
-    link_with: [felib, sqllib, fmtlib]
+    link_with: [felib, streamlib, sqllib, fmtlib]
 )
 fluxreaderinc = include_directories('lib/fluxreader')