Merge in fluxfile stuff.

README.md

@@ -263,6 +263,11 @@ As an exception, `dep/lexy` contains a partial copy of the lexy package, written
 by foonathen@github, taken from https://github.com/foonathan/lexy. It is BSL 1.0
 licensed. Please see the contents of the directory for the full text.
 
+As an exception, `dep/alphanum` contains a copy of the alphanum package,
+written by Dave Koelle, taken from
+https://web.archive.org/web/20210207124255/davekoelle.com/alphanum.html. It is
+MIT licensed. Please see the source for the full text.
+
 __Important:__ Because of all these exceptions, if you distribute the
 FluxEngine package as a whole, you must comply with the terms of _all_ of the
 licensing terms. This means that __effectively the FluxEngine package is

dep/alphanum/UPSTREAM.md

@@ -0,0 +1,2 @@
+Downloaded from:
+https://web.archive.org/web/20210918044134/http://davekoelle.com/files/alphanum.hpp

dep/alphanum/alphanum.h

@@ -0,0 +1,450 @@
+#ifndef ALPHANUM__HPP
+#define ALPHANUM__HPP
+
+/*
+ Released under the MIT License - https://opensource.org/licenses/MIT
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+/* $Header: /code/doj/alphanum.hpp,v 1.3 2008/01/28 23:06:47 doj Exp $ */
+
+#include <cassert>
+#include <functional>
+#include <string>
+#include <sstream>
+
+#ifdef ALPHANUM_LOCALE
+#include <cctype>
+#endif
+
+#ifdef DOJDEBUG
+#include <iostream>
+#include <typeinfo>
+#endif
+
+// TODO: make comparison with hexadecimal numbers. Extend the alphanum_comp() function by traits to choose between decimal and hexadecimal.
+
+namespace doj
+{
+
+  // anonymous namespace for functions we use internally. But if you
+  // are coding in C, you can use alphanum_impl() directly, since it
+  // uses not C++ features.
+  namespace {
+
+    // if you want to honour the locale settings for detecting digit
+    // characters, you should define ALPHANUM_LOCALE
+#ifdef ALPHANUM_LOCALE
+    /** wrapper function for ::isdigit() */
+    bool alphanum_isdigit(int c)
+    {
+      return isdigit(c);
+    }
+#else
+    /** this function does not consider the current locale and only
+	works with ASCII digits.
+	@return true if c is a digit character
+    */
+    bool alphanum_isdigit(const char c)
+    {
+      return c>='0' && c<='9';
+    }
+#endif
+
+    /**
+       compare l and r with strcmp() semantics, but using
+       the "Alphanum Algorithm". This function is designed to read
+       through the l and r strings only one time, for
+       maximum performance. It does not allocate memory for
+       substrings. It can either use the C-library functions isdigit()
+       and atoi() to honour your locale settings, when recognizing
+       digit characters when you "#define ALPHANUM_LOCALE=1" or use
+       it's own digit character handling which only works with ASCII
+       digit characters, but provides better performance.
+
+       @param l NULL-terminated C-style string
+       @param r NULL-terminated C-style string
+       @return negative if l<r, 0 if l equals r, positive if l>r
+     */
+    int alphanum_impl(const char *l, const char *r)
+    {
+      enum mode_t { STRING, NUMBER } mode=STRING;
+
+      while(*l && *r)
+	{
+	  if(mode == STRING)
+	    {
+	      char l_char, r_char;
+	      while((l_char=*l) && (r_char=*r))
+		{
+		  // check if this are digit characters
+		  const bool l_digit=alphanum_isdigit(l_char), r_digit=alphanum_isdigit(r_char);
+		  // if both characters are digits, we continue in NUMBER mode
+		  if(l_digit && r_digit)
+		    {
+		      mode=NUMBER;
+		      break;
+		    }
+		  // if only the left character is a digit, we have a result
+		  if(l_digit) return -1;
+		  // if only the right character is a digit, we have a result
+		  if(r_digit) return +1;
+		  // compute the difference of both characters
+		  const int diff=l_char - r_char;
+		  // if they differ we have a result
+		  if(diff != 0) return diff;
+		  // otherwise process the next characters
+		  ++l;
+		  ++r;
+		}
+	    }
+	  else // mode==NUMBER
+	    {
+#ifdef ALPHANUM_LOCALE
+	      // get the left number
+	      char *end;
+	      unsigned long l_int=strtoul(l, &end, 0);
+	      l=end;
+
+	      // get the right number
+	      unsigned long r_int=strtoul(r, &end, 0);
+	      r=end;
+#else
+	      // get the left number
+	      unsigned long l_int=0;
+	      while(*l && alphanum_isdigit(*l))
+		{
+		  // TODO: this can overflow
+		  l_int=l_int*10 + *l-'0';
+		  ++l;
+		}
+
+	      // get the right number
+	      unsigned long r_int=0;
+	      while(*r && alphanum_isdigit(*r))
+		{
+		  // TODO: this can overflow
+		  r_int=r_int*10 + *r-'0';
+		  ++r;
+		}
+#endif
+
+	      // if the difference is not equal to zero, we have a comparison result
+	      const long diff=l_int-r_int;
+	      if(diff != 0)
+		return diff;
+
+	      // otherwise we process the next substring in STRING mode
+	      mode=STRING;
+	    }
+	}
+
+      if(*r) return -1;
+      if(*l) return +1;
+      return 0;
+    }
+
+  }
+
+  /**
+     Compare left and right with the same semantics as strcmp(), but with the
+     "Alphanum Algorithm" which produces more human-friendly
+     results. The classes lT and rT must implement "std::ostream
+     operator<< (std::ostream&, const Ty&)".
+
+     @return negative if left<right, 0 if left==right, positive if left>right.
+  */
+  template <typename lT, typename rT>
+  int alphanum_comp(const lT& left, const rT& right)
+  {
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<" << typeid(left).name() << "," << typeid(right).name() << "> " << left << "," << right << std::endl;
+#endif
+    std::ostringstream l; l << left;
+    std::ostringstream r; r << right;
+    return alphanum_impl(l.str().c_str(), r.str().c_str());
+  }
+
+  /**
+     Compare l and r with the same semantics as strcmp(), but with
+     the "Alphanum Algorithm" which produces more human-friendly
+     results.
+
+     @return negative if l<r, 0 if l==r, positive if l>r.
+  */
+  template <>
+  int alphanum_comp<std::string>(const std::string& l, const std::string& r)
+  {
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<std::string,std::string> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l.c_str(), r.c_str());
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  // now follow a lot of overloaded alphanum_comp() functions to get a
+  // direct call to alphanum_impl() upon the various combinations of c
+  // and c++ strings.
+
+  /**
+     Compare l and r with the same semantics as strcmp(), but with
+     the "Alphanum Algorithm" which produces more human-friendly
+     results.
+
+     @return negative if l<r, 0 if l==r, positive if l>r.
+  */
+  int alphanum_comp(char* l, char* r)
+  {
+    assert(l);
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<char*,char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r);
+  }
+
+  int alphanum_comp(const char* l, const char* r)
+  {
+    assert(l);
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<const char*,const char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r);
+  }
+
+  int alphanum_comp(char* l, const char* r)
+  {
+    assert(l);
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<char*,const char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r);
+  }
+
+  int alphanum_comp(const char* l, char* r)
+  {
+    assert(l);
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<const char*,char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r);
+  }
+
+  int alphanum_comp(const std::string& l, char* r)
+  {
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<std::string,char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l.c_str(), r);
+  }
+
+  int alphanum_comp(char* l, const std::string& r)
+  {
+    assert(l);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<char*,std::string> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r.c_str());
+  }
+
+  int alphanum_comp(const std::string& l, const char* r)
+  {
+    assert(r);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<std::string,const char*> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l.c_str(), r);
+  }
+
+  int alphanum_comp(const char* l, const std::string& r)
+  {
+    assert(l);
+#ifdef DOJDEBUG
+    std::clog << "alphanum_comp<const char*,std::string> " << l << "," << r << std::endl;
+#endif
+    return alphanum_impl(l, r.c_str());
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+
+  /**
+     Functor class to compare two objects with the "Alphanum
+     Algorithm". If the objects are no std::string, they must
+     implement "std::ostream operator<< (std::ostream&, const Ty&)".
+  */
+  template<class Ty>
+  struct alphanum_less
+  {
+    bool operator()(const Ty& left, const Ty& right) const
+    {
+      return alphanum_comp(left, right) < 0;
+    }
+  };
+
+}
+
+#ifdef TESTMAIN
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+#include <map>
+#include <set>
+#include <vector>
+int main()
+{
+  // testcases for the algorithm
+  assert(doj::alphanum_comp("","") == 0);
+  assert(doj::alphanum_comp("","a") < 0);
+  assert(doj::alphanum_comp("a","") > 0);
+  assert(doj::alphanum_comp("a","a") == 0);
+  assert(doj::alphanum_comp("","9") < 0);
+  assert(doj::alphanum_comp("9","") > 0);
+  assert(doj::alphanum_comp("1","1") == 0);
+  assert(doj::alphanum_comp("1","2") < 0);
+  assert(doj::alphanum_comp("3","2") > 0);
+  assert(doj::alphanum_comp("a1","a1") == 0);
+  assert(doj::alphanum_comp("a1","a2") < 0);
+  assert(doj::alphanum_comp("a2","a1") > 0);
+  assert(doj::alphanum_comp("a1a2","a1a3") < 0);
+  assert(doj::alphanum_comp("a1a2","a1a0") > 0);
+  assert(doj::alphanum_comp("134","122") > 0);
+  assert(doj::alphanum_comp("12a3","12a3") == 0);
+  assert(doj::alphanum_comp("12a1","12a0") > 0);
+  assert(doj::alphanum_comp("12a1","12a2") < 0);
+  assert(doj::alphanum_comp("a","aa") < 0);
+  assert(doj::alphanum_comp("aaa","aa") > 0);
+  assert(doj::alphanum_comp("Alpha 2","Alpha 2") == 0);
+  assert(doj::alphanum_comp("Alpha 2","Alpha 2A") < 0);
+  assert(doj::alphanum_comp("Alpha 2 B","Alpha 2") > 0);
+
+  assert(doj::alphanum_comp(1,1) == 0);
+  assert(doj::alphanum_comp(1,2) < 0);
+  assert(doj::alphanum_comp(2,1) > 0);
+  assert(doj::alphanum_comp(1.2,3.14) < 0);
+  assert(doj::alphanum_comp(3.14,2.71) > 0);
+  assert(doj::alphanum_comp(true,true) == 0);
+  assert(doj::alphanum_comp(true,false) > 0);
+  assert(doj::alphanum_comp(false,true) < 0);
+
+  std::string str("Alpha 2");
+  assert(doj::alphanum_comp(str,"Alpha 2") == 0);
+  assert(doj::alphanum_comp(str,"Alpha 2A") < 0);
+  assert(doj::alphanum_comp("Alpha 2 B",str) > 0);
+
+  assert(doj::alphanum_comp(str,strdup("Alpha 2")) == 0);
+  assert(doj::alphanum_comp(str,strdup("Alpha 2A")) < 0);
+  assert(doj::alphanum_comp(strdup("Alpha 2 B"),str) > 0);
+
+#if 1
+  // show usage of the comparison functor with a set
+  std::set<std::string, doj::alphanum_less<std::string> > s;
+  s.insert("Xiph Xlater 58");
+  s.insert("Xiph Xlater 5000");
+  s.insert("Xiph Xlater 500");
+  s.insert("Xiph Xlater 50");
+  s.insert("Xiph Xlater 5");
+  s.insert("Xiph Xlater 40");
+  s.insert("Xiph Xlater 300");
+  s.insert("Xiph Xlater 2000");
+  s.insert("Xiph Xlater 10000");
+  s.insert("QRS-62F Intrinsia Machine");
+  s.insert("QRS-62 Intrinsia Machine");
+  s.insert("QRS-60F Intrinsia Machine");
+  s.insert("QRS-60 Intrinsia Machine");
+  s.insert("Callisto Morphamax 7000 SE2");
+  s.insert("Callisto Morphamax 7000 SE");
+  s.insert("Callisto Morphamax 7000");
+  s.insert("Callisto Morphamax 700");
+  s.insert("Callisto Morphamax 600");
+  s.insert("Callisto Morphamax 5000");
+  s.insert("Callisto Morphamax 500");
+  s.insert("Callisto Morphamax");
+  s.insert("Alpha 2A-900");
+  s.insert("Alpha 2A-8000");
+  s.insert("Alpha 2A");
+  s.insert("Alpha 200");
+  s.insert("Alpha 2");
+  s.insert("Alpha 100");
+  s.insert("Allegia 60 Clasteron");
+  s.insert("Allegia 52 Clasteron");
+  s.insert("Allegia 51B Clasteron");
+  s.insert("Allegia 51 Clasteron");
+  s.insert("Allegia 500 Clasteron");
+  s.insert("Allegia 50 Clasteron");
+  s.insert("40X Radonius");
+  s.insert("30X Radonius");
+  s.insert("20X Radonius Prime");
+  s.insert("20X Radonius");
+  s.insert("200X Radonius");
+  s.insert("10X Radonius");
+  s.insert("1000X Radonius Maximus");
+  // print sorted set to cout
+  std::copy(s.begin(), s.end(), std::ostream_iterator<std::string>(std::cout, "\n"));
+
+  // show usage of comparision functor with a map
+  typedef std::map<std::string, int, doj::alphanum_less<std::string> > m_t;
+  m_t m;
+  m["z1.doc"]=1;
+  m["z10.doc"]=2;
+  m["z100.doc"]=3;
+  m["z101.doc"]=4;
+  m["z102.doc"]=5;
+  m["z11.doc"]=6;
+  m["z12.doc"]=7;
+  m["z13.doc"]=8;
+  m["z14.doc"]=9;
+  m["z15.doc"]=10;
+  m["z16.doc"]=11;
+  m["z17.doc"]=12;
+  m["z18.doc"]=13;
+  m["z19.doc"]=14;
+  m["z2.doc"]=15;
+  m["z20.doc"]=16;
+  m["z3.doc"]=17;
+  m["z4.doc"]=18;
+  m["z5.doc"]=19;
+  m["z6.doc"]=20;
+  m["z7.doc"]=21;
+  m["z8.doc"]=22;
+  m["z9.doc"]=23;
+  // print sorted map to cout
+  for(m_t::iterator i=m.begin(); i!=m.end(); ++i)
+    std::cout << i->first << '\t' << i->second << std::endl;
+
+  // show usage of comparison functor with an STL algorithm on a vector
+  std::vector<std::string> v;
+  // vector contents are reversed sorted contents of the old set
+  std::copy(s.rbegin(), s.rend(), std::back_inserter(v));
+  // now sort the vector with the algorithm
+  std::sort(v.begin(), v.end(), doj::alphanum_less<std::string>());
+  // and print the vector to cout
+  std::copy(v.begin(), v.end(), std::ostream_iterator<std::string>(std::cout, "\n"));
+#endif
+
+  return 0;
+}
+#endif
+
+#endif

dep/alphanum/build.py

@@ -0,0 +1,8 @@
+from build.c import clibrary
+
+clibrary(
+    name="alphanum",
+    srcs=[],
+    hdrs={"dep/alphanum/alphanum.h": "./alphanum.h"},
+)
+

doc/using.md

@@ -377,6 +377,22 @@ FluxEngine also supports a number of file system image formats. When using the
   correctness. Individual records are separated by three `\\0` bytes and tracks
   are separated by four `\\0` bytes; tracks are emitted in CHS order.
 
+### Manipulating flux files
+
+`fluxengine fluxfile` contains a set of tools for examining or manipulating
+individual flux files. (Note that this only works on flux files themselves, not
+sources.)
+
+  - `fluxfile ls <fluxfile>`
+
+  Shows all the components inside a flux file.
+
+  - `fluxfile rm <fluxfile>:<path>...`
+
+  Removes flux from a flux file. You may specify the path to an individual read
+  (e.g. `track.h0_t5.flux9`) or the track itself (`track.h0_t5`); the latter
+  will remove all reads from the track.
+
 ### High density disks
 
 High density disks use a different magnetic medium to low and double density

lib/config/common.proto

@@ -2,9 +2,14 @@ syntax = "proto2";
 
 import "google/protobuf/descriptor.proto";
 
-extend google.protobuf.FieldOptions {
+extend google.protobuf.FieldOptions
-  optional string help = 50000;
+{
-  optional bool recurse = 50001 [default = true];
+    optional string help = 50000;
+}
+
+extend google.protobuf.MessageOptions
+{
+    optional bool recurse = 50001 [default = true];
 }
 
 enum IndexMode {
@@ -29,21 +34,18 @@ enum FluxSourceSinkType {
 	FLUXTYPE_DMK = 12;
 }
 
-enum ImageReaderWriterType {
+enum ImageReaderWriterType
-	IMAGETYPE_NOT_SET = 0;
+{
-	IMAGETYPE_D64 = 1;
+    IMAGETYPE_NOT_SET = 0; IMAGETYPE_D64 = 1; IMAGETYPE_D88 = 2;
-	IMAGETYPE_D88 = 2;
+    IMAGETYPE_DIM = 3;
-	IMAGETYPE_DIM = 3;
+    IMAGETYPE_DISKCOPY = 4;
-	IMAGETYPE_DISKCOPY = 4;
+    IMAGETYPE_FDI = 5;
-	IMAGETYPE_FDI = 5;
+    IMAGETYPE_IMD = 6;
-	IMAGETYPE_IMD = 6;
+    IMAGETYPE_IMG = 7;
-	IMAGETYPE_IMG = 7;
+    IMAGETYPE_JV3 = 8;
-	IMAGETYPE_JV3 = 8;
+    IMAGETYPE_LDBS = 9;
-	IMAGETYPE_LDBS = 9;
+    IMAGETYPE_NFD = 10;
-	IMAGETYPE_NFD = 10;
+    IMAGETYPE_NSI = 11;
-	IMAGETYPE_NSI = 11;
+    IMAGETYPE_RAW = 12;
-	IMAGETYPE_RAW = 12;
+    IMAGETYPE_TD0 = 13;
-	IMAGETYPE_TD0 = 13;
 }
-
-

lib/config/config.proto

@@ -14,20 +14,20 @@ import "lib/config/layout.proto";
 
 enum SupportStatus
 {
-    UNSUPPORTED = 0;
+    UNSUPPORTED = 0; DINOSAUR = 1; UNICORN = 2;
-    DINOSAUR = 1;
-    UNICORN = 2;
 }
 
 // NEXT_TAG: 27
 message ConfigProto
 {
+    option(recurse) = false;
+
     optional string shortname = 1;
     optional string comment = 2;
     optional bool is_extension = 3;
     repeated string documentation = 4;
-    optional SupportStatus read_support_status = 5 [ default = UNSUPPORTED ];
+    optional SupportStatus read_support_status = 5 [default = UNSUPPORTED];
-    optional SupportStatus write_support_status = 6 [ default = UNSUPPORTED ];
+    optional SupportStatus write_support_status = 6 [default = UNSUPPORTED];
 
     optional LayoutProto layout = 7;
 
@@ -51,28 +51,27 @@ message ConfigProto
 
 message OptionPrerequisiteProto
 {
-    optional string key = 1 [ (help) = "path to config value" ];
+    optional string key = 1 [(help) = "path to config value"];
-    repeated string value = 2 [ (help) = "list of required values" ];
+    repeated string value = 2 [(help) = "list of required values"];
 }
 
 // NEXT_TAG: 8
 message OptionProto
 {
-    optional string name = 1 [ (help) = "option name" ];
+    optional string name = 1 [(help) = "option name"];
-    optional string comment = 2 [ (help) = "help text for option" ];
+    optional string comment = 2 [(help) = "help text for option"];
-    optional string message = 3
+    optional string message =
-        [ (help) = "message to display when option is in use" ];
+        3 [(help) = "message to display when option is in use"];
-    optional bool set_by_default = 6
+    optional bool set_by_default =
-        [ (help) = "this option is applied by default", default = false ];
+        6 [(help) = "this option is applied by default", default = false];
-    repeated OptionPrerequisiteProto prerequisite = 7
+    repeated OptionPrerequisiteProto prerequisite =
-        [ (help) = "prerequisites for this option" ];
+        7 [(help) = "prerequisites for this option"];
 
-    optional ConfigProto config = 4
+    optional ConfigProto config = 4 [(help) = "option data"];
-        [ (help) = "option data", (recurse) = false ];
 }
 
 message OptionGroupProto
 {
-    optional string comment = 1 [ (help) = "help text for option group" ];
+    optional string comment = 1 [(help) = "help text for option group"];
     repeated OptionProto option = 2;
 }

lib/config/flags.cc

@@ -264,7 +264,7 @@ static void doShowConfig()
 
 static void doDoc()
 {
-    const auto fields = findAllProtoFields(globalConfig().base());
+    const auto fields = findAllPossibleProtoFields(globalConfig().base()->GetDescriptor());
     for (const auto field : fields)
     {
         const std::string& path = field.first;

lib/config/proto.cc

@@ -1,6 +1,7 @@
 #include "lib/core/globals.h"
 #include "lib/config/proto.h"
 #include "lib/config/common.pb.h"
+#include "google/protobuf/reflection.h"
 #include <regex>
 
 static ConfigProto config = []()
@@ -16,7 +17,7 @@ ConfigProto& globalConfigProto()
     return config;
 }
 
-static double toFloat(const std::string& value)
+static float toFloat(const std::string& value)
 {
     try
     {
@@ -66,6 +67,21 @@ static uint64_t toUint64(const std::string& value)
     return d;
 }
 
+static int splitIndexedField(std::string& item)
+{
+    static const std::regex INDEX_REGEX("(\\w+)\\[([0-9]+)\\]");
+    int index = -1;
+
+    std::smatch dmatch;
+    if (std::regex_match(item, dmatch, INDEX_REGEX))
+    {
+        item = dmatch[1];
+        index = std::stoi(dmatch[2]);
+    }
+
+    return index;
+}
+
 static ProtoField resolveProtoPath(
     google::protobuf::Message* message, const std::string& path, bool create)
 {
@@ -86,6 +102,10 @@ static ProtoField resolveProtoPath(
     std::stringstream ss(leading);
     while (std::getline(ss, item, '.'))
     {
+        static const std::regex INDEX_REGEX("(\\w+)\\[([0-9]+)\\]");
+
+        int index = splitIndexedField(item);
+
         const auto* field = descriptor->FindFieldByName(item);
         if (!field)
             throw ProtoPathNotFoundException(
@@ -95,6 +115,14 @@ static ProtoField resolveProtoPath(
                 "config field '{}' in '{}' is not a message", item, path));
 
         const auto* reflection = message->GetReflection();
+        if ((field->label() !=
+                google::protobuf::FieldDescriptor::LABEL_REPEATED) &&
+            (index != -1))
+            throw ProtoPathNotFoundException(fmt::format(
+                "config field '{}[{}]' is indexed, but not repeated",
+                item,
+                index));
+
         switch (field->label())
         {
             case google::protobuf::FieldDescriptor::LABEL_OPTIONAL:
@@ -106,16 +134,15 @@ static ProtoField resolveProtoPath(
                 break;
 
             case google::protobuf::FieldDescriptor::LABEL_REPEATED:
-                if (reflection->FieldSize(*message, field) == 0)
+                if (index == -1)
-                {
+                    throw ProtoPathNotFoundException(fmt::format(
-                    if (create)
+                        "config field '{}' is repeated and must be indexed",
-                        message = reflection->AddMessage(message, field);
+                        item));
-                    else
+                while (reflection->FieldSize(*message, field) <= index)
-                        fail();
+                    reflection->AddMessage(message, field);
-                }
+
-                else
+                message =
-                    message =
+                    reflection->MutableRepeatedMessage(message, field, index);
-                        reflection->MutableRepeatedMessage(message, field, 0);
                 break;
 
             default:
@@ -125,11 +152,12 @@ static ProtoField resolveProtoPath(
         descriptor = message->GetDescriptor();
     }
 
+    int index = splitIndexedField(trailing);
     const auto* field = descriptor->FindFieldByName(trailing);
     if (!field)
         fail();
 
-    return std::make_pair(message, field);
+    return std::make_tuple(message, field, index);
 }
 
 ProtoField makeProtoPath(
@@ -144,91 +172,206 @@ ProtoField findProtoPath(
     return resolveProtoPath(message, path, /* create= */ false);
 }
 
+static bool parseBoolean(const std::string& value)
+{
+    static const std::map<std::string, bool> boolvalues = {
+        {"false", false},
+        {"f",     false},
+        {"no",    false},
+        {"n",     false},
+        {"0",     false},
+        {"true",  true },
+        {"t",     true },
+        {"yes",   true },
+        {"y",     true },
+        {"1",     true },
+    };
+
+    const auto& it = boolvalues.find(value);
+    if (it == boolvalues.end())
+        error("invalid boolean value");
+    return it->second;
+}
+
+static int32_t parseEnum(
+    const google::protobuf::FieldDescriptor* field, const std::string& value)
+{
+    const auto* enumfield = field->enum_type();
+    const auto* enumvalue = enumfield->FindValueByName(value);
+    if (!enumvalue)
+        error("unrecognised enum value '{}'", value);
+    return enumvalue->number();
+}
+
+template <typename T>
+static void updateRepeatedField(
+    google::protobuf::MutableRepeatedFieldRef<T> mrfr, int index, T value)
+{
+    mrfr.Set(index, value);
+}
+
 void setProtoFieldFromString(ProtoField& protoField, const std::string& value)
 {
-    google::protobuf::Message* message = protoField.first;
+    auto& [message, field, index] = protoField;
-    const google::protobuf::FieldDescriptor* field = protoField.second;
 
     const auto* reflection = message->GetReflection();
-    switch (field->type())
+    if (field->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
     {
-        case google::protobuf::FieldDescriptor::TYPE_FLOAT:
+        if (index == -1)
-            reflection->SetFloat(message, field, toFloat(value));
+            error("field '{}' is repeated but no index is provided");
-            break;
 
-        case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
+        switch (field->type())
-            reflection->SetDouble(message, field, toDouble(value));
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_INT32:
-            reflection->SetInt32(message, field, toInt64(value));
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_INT64:
-            reflection->SetInt64(message, field, toInt64(value));
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_UINT32:
-            reflection->SetUInt32(message, field, toUint64(value));
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_UINT64:
-            reflection->SetUInt64(message, field, toUint64(value));
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_STRING:
-            reflection->SetString(message, field, value);
-            break;
-
-        case google::protobuf::FieldDescriptor::TYPE_BOOL:
         {
-            static const std::map<std::string, bool> boolvalues = {
+            case google::protobuf::FieldDescriptor::TYPE_FLOAT:
-                {"false", false},
+                updateRepeatedField(
-                {"f",     false},
+                    reflection->GetMutableRepeatedFieldRef<float>(
-                {"no",    false},
+                        message, field),
-                {"n",     false},
+                    index,
-                {"0",     false},
+                    toFloat(value));
-                {"true",  true },
-                {"t",     true },
-                {"yes",   true },
-                {"y",     true },
-                {"1",     true },
-            };
-
-            const auto& it = boolvalues.find(value);
-            if (it == boolvalues.end())
-                error("invalid boolean value");
-            reflection->SetBool(message, field, it->second);
-            break;
-        }
-
-        case google::protobuf::FieldDescriptor::TYPE_ENUM:
-        {
-            const auto* enumfield = field->enum_type();
-            const auto* enumvalue = enumfield->FindValueByName(value);
-            if (!enumvalue)
-                error("unrecognised enum value '{}'", value);
-
-            reflection->SetEnum(message, field, enumvalue);
-            break;
-        }
-
-        case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
-            if (field->containing_oneof() && value.empty())
-            {
-                reflection->MutableMessage(message, field);
                 break;
-            }
+
-            /* fall through */
+            case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
-        default:
+                updateRepeatedField(
-            error("can't set this config value type");
+                    reflection->GetMutableRepeatedFieldRef<double>(
+                        message, field),
+                    index,
+                    toDouble(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_INT32:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<int32_t>(
+                        message, field),
+                    index,
+                    (int32_t)toInt64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_INT64:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<int64_t>(
+                        message, field),
+                    index,
+                    toInt64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_UINT32:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<uint32_t>(
+                        message, field),
+                    index,
+                    (uint32_t)toUint64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_UINT64:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<uint64_t>(
+                        message, field),
+                    index,
+                    toUint64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_STRING:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<std::string>(
+                        message, field),
+                    index,
+                    value);
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_BOOL:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<bool>(
+                        message, field),
+                    index,
+                    parseBoolean(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_ENUM:
+                updateRepeatedField(
+                    reflection->GetMutableRepeatedFieldRef<int32_t>(
+                        message, field),
+                    index,
+                    parseEnum(field, value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
+                if (field->message_type() == RangeProto::descriptor())
+                {
+                    setRange((RangeProto*)reflection->MutableRepeatedMessage(
+                                 message, field, index),
+                        value);
+                    break;
+                }
+                /* fall through */
+            default:
+                error("can't set this config value type");
+        }
+    }
+    else
+    {
+        if (index != -1)
+            error("field '{}' is not repeated but an index is provided");
+        switch (field->type())
+        {
+            case google::protobuf::FieldDescriptor::TYPE_FLOAT:
+                reflection->SetFloat(message, field, toFloat(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_DOUBLE:
+                reflection->SetDouble(message, field, toDouble(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_INT32:
+                reflection->SetInt32(message, field, toInt64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_INT64:
+                reflection->SetInt64(message, field, toInt64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_UINT32:
+                reflection->SetUInt32(message, field, toUint64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_UINT64:
+                reflection->SetUInt64(message, field, toUint64(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_STRING:
+                reflection->SetString(message, field, value);
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_BOOL:
+                reflection->SetBool(message, field, parseBoolean(value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_ENUM:
+                reflection->SetEnumValue(message, field, parseEnum(field, value));
+                break;
+
+            case google::protobuf::FieldDescriptor::TYPE_MESSAGE:
+                if (field->message_type() == RangeProto::descriptor())
+                {
+                    setRange(
+                        (RangeProto*)reflection->MutableMessage(message, field),
+                        value);
+                    break;
+                }
+                if (field->containing_oneof() && value.empty())
+                {
+                    reflection->MutableMessage(message, field);
+                    break;
+                }
+                /* fall through */
+            default:
+                error("can't set this config value type");
+        }
     }
 }
 
 std::string getProtoFieldValue(ProtoField& protoField)
 {
-    google::protobuf::Message* message = protoField.first;
+    auto& [message, field, index] = protoField;
-    const google::protobuf::FieldDescriptor* field = protoField.second;
 
     const auto* reflection = message->GetReflection();
     switch (field->type())
@@ -294,11 +437,17 @@ std::set<unsigned> iterate(unsigned start, unsigned count)
     return set;
 }
 
+static bool shouldRecurse(const google::protobuf::FieldDescriptor* f)
+{
+    if (f->type() != google::protobuf::FieldDescriptor::TYPE_MESSAGE)
+        return false;
+    return f->message_type()->options().GetExtension(::recurse);
+}
+
 std::map<std::string, const google::protobuf::FieldDescriptor*>
-findAllProtoFields(google::protobuf::Message* message)
+findAllPossibleProtoFields(const google::protobuf::Descriptor* descriptor)
 {
     std::map<std::string, const google::protobuf::FieldDescriptor*> fields;
-    const auto* descriptor = message->GetDescriptor();
 
     std::function<void(const google::protobuf::Descriptor*, const std::string&)>
         recurse = [&](auto* d, const auto& s)
@@ -308,8 +457,10 @@ findAllProtoFields(google::protobuf::Message* message)
             const google::protobuf::FieldDescriptor* f = d->field(i);
             std::string n = s + f->name();
 
-            if (f->options().GetExtension(::recurse) &&
+            if (f->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
-                (f->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE))
+                n += "[]";
+
+            if (shouldRecurse(f))
                 recurse(f->message_type(), n + ".");
 
             fields[n] = f;
@@ -320,10 +471,47 @@ findAllProtoFields(google::protobuf::Message* message)
     return fields;
 }
 
-ConfigProto parseConfigBytes(const std::string_view& data)
+std::map<std::string, const google::protobuf::FieldDescriptor*>
+findAllProtoFields(const google::protobuf::Message& message)
 {
-    ConfigProto proto;
+    std::map<std::string, const google::protobuf::FieldDescriptor*> allFields;
-    if (!proto.ParseFromArray(data.begin(), data.size()))
+
-        error("invalid internal config data");
+    std::function<void(const google::protobuf::Message&, const std::string&)>
-    return proto;
+        recurse = [&](auto& message, const auto& name)
+    {
+        const auto* reflection = message.GetReflection();
+        std::vector<const google::protobuf::FieldDescriptor*> fields;
+        reflection->ListFields(message, &fields);
+
+        for (const auto* f : fields)
+        {
+            auto basename = name;
+            if (!basename.empty())
+                basename += '.';
+            basename += f->name();
+
+            if (f->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
+            {
+                for (int i = 0; i < reflection->FieldSize(message, f); i++)
+                {
+                    const auto n = fmt::format("{}[{}]", basename, i);
+                    if (shouldRecurse(f))
+                        recurse(
+                            reflection->GetRepeatedMessage(message, f, i), n);
+                    else
+                        allFields[n] = f;
+                }
+            }
+            else
+            {
+                if (shouldRecurse(f))
+                    recurse(reflection->GetMessage(message, f), basename);
+                else
+                    allFields[basename] = f;
+            }
+        }
+    };
+
+    recurse(message, "");
+    return allFields;
 }

lib/config/proto.h

@@ -14,8 +14,8 @@ public:
     }
 };
 
-typedef std::pair<google::protobuf::Message*,
+typedef std::tuple<google::protobuf::Message*,
-    const google::protobuf::FieldDescriptor*>
+    const google::protobuf::FieldDescriptor*, int>
     ProtoField;
 
 extern ProtoField makeProtoPath(
@@ -34,9 +34,19 @@ extern std::string getProtoByString(
 extern std::set<unsigned> iterate(unsigned start, unsigned count);
 
 extern std::map<std::string, const google::protobuf::FieldDescriptor*>
-findAllProtoFields(google::protobuf::Message* message);
+findAllPossibleProtoFields(const google::protobuf::Descriptor* descriptor);
 
-extern ConfigProto parseConfigBytes(const std::string_view& bytes);
+extern std::map<std::string, const google::protobuf::FieldDescriptor*>
+findAllProtoFields(const google::protobuf::Message& message);
+
+template <class T>
+static inline const T parseProtoBytes(const std::string_view& bytes)
+{
+    T proto;
+    if (!proto.ParseFromArray(bytes.begin(), bytes.size()))
+        error("invalid internal proto data");
+    return proto;
+}
 
 extern const std::map<std::string, const ConfigProto*> formats;
 

lib/core/globals.h

@@ -24,6 +24,9 @@
 #define mkdir(A, B) _mkdir(A)
 #endif
 
+#define STRINGIFY(a) XSTRINGIFY(a)
+#define XSTRINGIFY(a) #a
+
 template <class T>
 static inline std::vector<T> vector_of(T item)
 {

lib/external/fl2.proto

@@ -1,5 +1,12 @@
 syntax = "proto2";
 
+import "google/protobuf/descriptor.proto";
+
+extend google.protobuf.FieldOptions
+{
+    optional bool isflux = 60000 [default = false];
+}
+
 enum FluxMagic {
 	MAGIC = 0x466c7578;
 }
@@ -12,7 +19,7 @@ enum FluxFileVersion {
 message TrackFluxProto {
 	optional int32 track = 1;
 	optional int32 head = 2;
-	repeated bytes flux = 3;
+	repeated bytes flux = 3 [(isflux) = true];
 }
 
 enum DriveType {

scripts/build.py

@@ -5,13 +5,14 @@ encoders = {}
 
 
 @Rule
-def protoencode_single(self, name, srcs: Targets, proto, symbol):
+def protoencode_single(self, name, srcs: Targets, proto, include, symbol):
     if proto not in encoders:
         r = cxxprogram(
             name="protoencode_" + proto,
             srcs=["scripts/protoencode.cc"],
-            cflags=["-DPROTO=" + proto],
+            cflags=["-DPROTO=" + proto, "-DINCLUDE="+include],
             deps=[
+                "lib/core",
                 "lib/config+proto_lib",
                 "lib/fluxsource+proto_lib",
                 "lib/fluxsink+proto_lib",
@@ -41,12 +42,13 @@ def protoencode_single(self, name, srcs: Targets, proto, symbol):
 
 
 @Rule
-def protoencode(self, name, proto, srcs: TargetsMap, symbol):
+def protoencode(self, name, proto, include,srcs: TargetsMap, symbol):
     encoded = [
         protoencode_single(
             name=f"{k}_cc",
             srcs=[v],
             proto=proto,
+            include=include,
             symbol=f"{symbol}_{k}_pb",
         )
         for k, v in srcs.items()

scripts/protoencode.cc

@@ -3,12 +3,13 @@
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <fstream>
 #include "fmt/format.h"
+#include "lib/core/globals.h"
 #include "tests/testproto.pb.h"
 #include "lib/config/config.pb.h"
 #include <sstream>
 #include <locale>
 
-#define STRINGIFY(s) #s
+const std::string protoname = STRINGIFY(PROTO);
 
 static uint32_t readu8(std::string::iterator& it, std::string::iterator end)
 {
@@ -125,6 +126,7 @@ int main(int argc, const char* argv[])
 
     output << "#include \"lib/core/globals.h\"\n"
            << "#include \"lib/config/proto.h\"\n"
+           << "#include \"" STRINGIFY(INCLUDE) "\"\n"
            << "#include <string_view>\n"
            << "static const uint8_t " << name << "_rawData[] = {";
 
@@ -143,11 +145,11 @@ int main(int argc, const char* argv[])
     output << "\n};\n";
     output << "extern const std::string_view " << name << "_data;\n";
     output << "const std::string_view " << name
-           << "_data = std::string_view((const char*)" << name << "_rawData, " << data.size()
+           << "_data = std::string_view((const char*)" << name << "_rawData, "
-           << ");\n";
+           << data.size() << ");\n";
-    output << "extern const ConfigProto " << name << ";\n";
+    output << "extern const " << protoname << " " << name << ";\n";
-    output << "const ConfigProto " << name << " = parseConfigBytes("
+    output << "const " << protoname << " " << name << " = parseProtoBytes<"
-           << argv[3] << "_data);\n";
+           << protoname << ">(" << argv[3] << "_data);\n";
 
     return 0;
 }

src/build.py

@@ -5,10 +5,14 @@ cxxprogram(
     srcs=[
         "./fluxengine.cc",
         "./fluxengine.h",
+        "./fluxfile.cc",
+        "./fluxfile.h",
         "./fe-analysedriveresponse.cc",
         "./fe-analyselayout.cc",
         "./fe-convert.cc",
         "./fe-format.cc",
+        "./fe-fluxfilels.cc",
+        "./fe-fluxfilerm.cc",
         "./fe-getdiskinfo.cc",
         "./fe-getfile.cc",
         "./fe-getfileinfo.cc",
@@ -39,6 +43,7 @@ cxxprogram(
         "+z_lib",
         "dep/adflib",
         "dep/agg",
+        "dep/alphanum",
         "dep/fatfs",
         "dep/hfsutils",
         "dep/libusbp",

src/fe-fluxfilels.cc

@@ -0,0 +1,39 @@
+#include "lib/core/globals.h"
+#include "lib/config/flags.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/sector.h"
+#include "lib/config/proto.h"
+#include "lib/data/flux.h"
+#include "lib/external/fl2.h"
+#include "lib/external/fl2.pb.h"
+#include "dep/alphanum/alphanum.h"
+#include "src/fluxengine.h"
+#include <fstream>
+
+static FlagGroup flags;
+static std::string filename;
+
+int mainFluxfileLs(int argc, const char* argv[])
+{
+    const auto filenames = flags.parseFlagsWithFilenames(argc, argv);
+    if (filenames.size() != 1)
+        error("you must specify exactly one filename");
+
+    for (const auto& filename : filenames)
+    {
+        fmt::print("Contents of {}:\n", filename);
+        FluxFileProto f = loadFl2File(filename);
+
+        auto fields = findAllProtoFields(f);
+		std::set<std::string, doj::alphanum_less<std::string>> fieldsSorted;
+        for (const auto& e : fields)
+			fieldsSorted.insert(e.first);
+
+        for (const auto& e : fieldsSorted)
+        {
+            fmt::print("{}: {}\n", e, getProtoByString(&f, e));
+        }
+    }
+
+    return 0;
+}

src/fe-fluxfilerm.cc

@@ -0,0 +1,47 @@
+#include "lib/core/globals.h"
+#include "lib/config/flags.h"
+#include "lib/data/fluxmap.h"
+#include "lib/data/sector.h"
+#include "lib/config/proto.h"
+#include "lib/data/flux.h"
+#include "lib/external/fl2.h"
+#include "lib/external/fl2.pb.h"
+#include "src/fluxengine.h"
+#include <fstream>
+
+static FlagGroup flags;
+static std::string filename;
+
+int mainFluxfileRm(int argc, const char* argv[])
+{
+    const auto filenames = flags.parseFlagsWithFilenames(argc, argv);
+    if (filenames.size() != 1)
+        error("you must specify exactly one filename");
+
+    const auto& filename = *filenames.begin();
+    fmt::print("Contents of {}:\n", filename);
+    FluxFileProto f = loadFl2File(filename);
+
+    fmt::print("version: {}\n", getProtoByString(&f, "version"));
+    fmt::print("rotational_period_ms: {}\n",
+        getProtoByString(&f, "rotational_period_ms"));
+    fmt::print("drive_type: {}\n", getProtoByString(&f, "drive_type"));
+    fmt::print("format_type: {}\n", getProtoByString(&f, "format_type"));
+    for (const auto& track : f.track())
+    {
+        for (int i = 0; i < track.flux().size(); i++)
+        {
+            const auto& flux = track.flux().at(i);
+            Fluxmap fluxmap(flux);
+
+            fmt::print("track.t{}_h{}.flux{}: {:.3f} ms, {} bytes\n",
+                track.track(),
+                track.head(),
+                i,
+                fluxmap.duration() / 1000000,
+                fluxmap.bytes());
+        }
+    }
+
+    return 0;
+}

src/fluxengine.cc

@@ -7,6 +7,8 @@ typedef int command_cb(int agrc, const char* argv[]);
 extern command_cb mainAnalyseDriveResponse;
 extern command_cb mainAnalyseLayout;
 extern command_cb mainConvert;
+extern command_cb mainFluxfileLs;
+extern command_cb mainFluxfileRm;
 extern command_cb mainFormat;
 extern command_cb mainGetDiskInfo;
 extern command_cb mainGetFile;
@@ -36,6 +38,7 @@ struct Command
 };
 
 static command_cb mainAnalyse;
+static command_cb mainFluxfile;
 static command_cb mainTest;
 
 // clang-format off
@@ -45,6 +48,7 @@ static std::vector<Command> commands =
 	{ "analyse",           mainAnalyse,           "Disk and drive analysis tools." },
     { "read",              mainRead,              "Reads a disk, producing a sector image.", },
     { "write",             mainWrite,             "Writes a sector image to a disk.", },
+	{ "fluxfile",          mainFluxfile,          "Flux file manipulation operations.", },
 	{ "format",            mainFormat,            "Format a disk and make a file system on it.", },
 	{ "rawread",           mainRawRead,           "Reads raw flux from a disk. Warning: you can't use this to copy disks.", },
     { "rawwrite",          mainRawWrite,          "Writes a flux file to a disk. Warning: you can't use this to copy disks.", },
@@ -71,9 +75,15 @@ static std::vector<Command> analysables =
 
 static std::vector<Command> testables =
 {
-    { "bandwidth",     mainTestBandwidth, "Measures your USB bandwidth.", },
+    { "bandwidth",     mainTestBandwidth,        "Measures your USB bandwidth.", },
-    { "devices",       mainTestDevices, "Displays all detected devices.", },
+    { "devices",       mainTestDevices,          "Displays all detected devices.", },
-    { "voltages",      mainTestVoltages,  "Measures the FDD bus voltages.", },
+    { "voltages",      mainTestVoltages,         "Measures the FDD bus voltages.", },
+};
+
+static std::vector<Command> fluxfileables =
+{
+    { "ls",            mainFluxfileLs,           "Lists the contents of a flux file.", },
+    { "rm",            mainFluxfileRm,           "Removes flux from a flux file.", },
 };
 // clang-format on
 
@@ -122,6 +132,11 @@ static int mainTest(int argc, const char* argv[])
     return mainExtended(testables, "test", argc, argv);
 }
 
+static int mainFluxfile(int argc, const char* argv[])
+{
+    return mainExtended(fluxfileables, "fluxfile", argc, argv);
+}
+
 static void globalHelp()
 {
     std::cout << "fluxengine: syntax: fluxengine <command> [<flags>...]\n"

src/fluxfile.h

@@ -0,0 +1,4 @@
+#pragma once
+
+extern void parseFluxfilePath(
+    const std::string& combined, std::string& filename, std::string& path);

src/formats/build.py

@@ -58,6 +58,7 @@ protoencode(
     name="formats_cc",
     srcs={name: f"./{name}.textpb" for name in formats},
     proto="ConfigProto",
+    include="lib/config/config.pb.h",
     symbol="formats",
 )
 

src/gui/drivetypes/build.py

@@ -26,6 +26,7 @@ protoencode(
     name="drivetypes_cc",
     srcs={name: f"./{name}.textpb" for name in drivetypes},
     proto="ConfigProto",
+    include="lib/config/config.pb.h",
     symbol="drivetypes",
 )
 

tests/build.py

@@ -40,44 +40,38 @@ tests = [
     "vfs",
 ]
 
+protoencode_single(
+    name="testproto_cc",
+    srcs=["./testproto.textpb"],
+    proto="TestProto",
+    include="tests/testproto.pb.h",
+    symbol="testproto_pb",
+)
+
 export(
     name="tests",
     deps=[
         test(
-            name="proto_test",
+            name=f"{n}_test",
             command=cxxprogram(
-                name="proto_test_exe",
+                name=f"{n}_test_exe",
                 srcs=[
-                    "./proto.cc",
+                    f"./{n}.cc",
-                    protoencode_single(
+                    ".+testproto_cc",
-                        name="testproto_cc",
-                        srcs=["./testproto.textpb"],
-                        proto="TestProto",
-                        symbol="testproto_pb",
-                    ),
                 ],
                 deps=[
-                    "lib/external+fl2_proto_lib",
                     "+fmt_lib",
                     "+protobuf_lib",
                     "+protocol",
-                    "+z_lib",
                     ".+test_proto_lib",
-                    "dep/adflib",
-                    "dep/agg",
-                    "dep/fatfs",
-                    "dep/hfsutils",
-                    "dep/libusbp",
                     "dep/snowhouse",
-                    "dep/stb",
                     "lib/config",
                     "lib/core",
-                    "lib/data",
                     "lib/fluxsource+proto_lib",
-                    "src/formats",
                 ],
             ),
-        ),
+        )
+        for n in ["proto"]
     ]
     + [
         test(

tests/proto.cc

@@ -28,8 +28,9 @@ static void test_setting(void)
     setProtoByString(&config, "d", "5.5");
     setProtoByString(&config, "f", "6.7");
     setProtoByString(&config, "m.s", "string");
-    setProtoByString(&config, "r.s", "val1");
+    setProtoByString(&config, "r[0].s", "val1");
-    setProtoByString(&config, "r.s", "val2");
+    setProtoByString(&config, "r[0].s", "val2");
+    setProtoByString(&config, "r[1].s", "val3");
     setProtoByString(&config, "firstoption.s", "1");
     setProtoByString(&config, "secondoption.s", "2");
 
@@ -48,6 +49,9 @@ static void test_setting(void)
 		r {
 			s: "val2"
 		}
+		r {
+			s: "val3"
+		}
 		secondoption {
 			s: "2"
 		}
@@ -70,8 +74,14 @@ static void test_getting(void)
 		r {
 			s: "val2"
 		}
+		r {
+			s: "val3"
+		}
 		secondoption {
 			s: "2"
+			r: 0
+			r: 1
+			r: 2
 		}
 	)M";
 
@@ -86,9 +96,11 @@ static void test_getting(void)
     AssertThat(getProtoByString(&tp, "d"), Equals("5.5"));
     AssertThat(getProtoByString(&tp, "f"), Equals("6.7"));
     AssertThat(getProtoByString(&tp, "m.s"), Equals("string"));
-    AssertThat(getProtoByString(&tp, "r.s"), Equals("val2"));
+    AssertThat(getProtoByString(&tp, "r[0].s"), Equals("val2"));
+    AssertThat(getProtoByString(&tp, "r[1].s"), Equals("val3"));
     AssertThrows(
         ProtoPathNotFoundException, getProtoByString(&tp, "firstoption.s"));
+    AssertThat(getProtoByString(&tp, "secondoption.r[2]"), Equals("2"));
     AssertThat(getProtoByString(&tp, "secondoption.s"), Equals("2"));
 }
 
@@ -131,8 +143,31 @@ static void test_load(void)
 static void test_fields(void)
 {
     TestProto proto;
-    auto fields = findAllProtoFields(&proto);
+    auto fields = findAllPossibleProtoFields(proto.GetDescriptor());
-    AssertThat(fields.size(), Equals(14));
+    std::vector<std::string> fieldNames;
+    for (const auto& e : fields)
+        fieldNames.push_back(e.first);
+
+    AssertThat(fieldNames,
+        Equals(std::vector<std::string>{"d",
+            "f",
+            "firstoption",
+            "firstoption.r[]",
+            "firstoption.s",
+            "i32",
+            "i64",
+            "m",
+			"m.r[]",
+            "m.s",
+            "r[]",
+            "r[].r[]",
+            "r[].s",
+            "range",
+            "secondoption",
+            "secondoption.r[]",
+            "secondoption.s",
+            "u32",
+            "u64"}));
 }
 
 static void test_options(void)
@@ -145,6 +180,57 @@ static void test_options(void)
     AssertThat(s, Equals("i64"));
 }
 
+static void test_findallfields(void)
+{
+    std::string s = R"M(
+		i64: -1
+		i32: -2
+		u64: 3
+		u32: 4
+		d: 5.5
+		f: 6.7
+		m {
+			s: "string"
+		}
+		r {
+			s: "val2"
+		}
+		r {
+			s: "val3"
+		}
+		secondoption {
+			s: "2"
+		}
+		range {
+			start: 1
+			step: 2
+			end: 3
+		}
+	)M";
+
+    TestProto proto;
+    if (!google::protobuf::TextFormat::MergeFromString(cleanup(s), &proto))
+        error("couldn't load test proto");
+
+    auto fields = findAllProtoFields(proto);
+    std::vector<std::string> fieldNames;
+    for (const auto& e : fields)
+        fieldNames.push_back(e.first);
+
+    AssertThat(fieldNames,
+        Equals(std::vector<std::string>{"d",
+            "f",
+            "i32",
+            "i64",
+            "m.s",
+            "r[0].s",
+            "r[1].s",
+            "range",
+            "secondoption.s",
+            "u32",
+            "u64"}));
+}
+
 int main(int argc, const char* argv[])
 {
     try
@@ -155,6 +241,7 @@ int main(int argc, const char* argv[])
         test_load();
         test_fields();
         test_options();
+        test_findallfields();
     }
     catch (const ErrorException& e)
     {

tests/testproto.proto

@@ -2,23 +2,25 @@ syntax = "proto2";
 
 import "lib/config/common.proto";
 
-message TestProto {
+message TestProto
-	message SubMessageProto {
+{
-		optional string s = 1;
+    message SubMessageProto
-	}
+    {
+        optional string s = 1;
+		repeated int32 r = 2;
+    }
 
-	optional int64 i64 = 1 [(help)="i64"];
+    optional int64 i64 = 1 [(help) = "i64"];
-	optional int32 i32 = 2;
+    optional int32 i32 = 2;
-	optional uint64 u64 = 3;
+    optional uint64 u64 = 3;
-	optional uint32 u32 = 4;
+    optional uint32 u32 = 4;
-	optional double d = 5;
+    optional double d = 5;
-	optional double f = 11;
+    optional double f = 11;
-	optional SubMessageProto m = 6;
+    optional SubMessageProto m = 6;
-	repeated SubMessageProto r = 7;
+    repeated SubMessageProto r = 7;
 
 	oneof alt {
 		SubMessageProto firstoption = 8;
 		SubMessageProto secondoption = 9;
 	}
 }
-