diff --git a/doc/using.md b/doc/using.md index 7ea81f55..a18ef5a9 100644 --- a/doc/using.md +++ b/doc/using.md @@ -155,6 +155,14 @@ more common tools. encoding. You can specify a profile if you want to write a subset of the disk. + - `fluxengine merge -s -s -d -c -h -B` Reads flux (possibly from a disk) and does various analyses of it to try diff --git a/lib/build.mk b/lib/build.mk index 3c6f84d7..2f640c5a 100644 --- a/lib/build.mk +++ b/lib/build.mk @@ -9,6 +9,7 @@ LIBFLUXENGINE_SRCS = \ lib/decoders/fmmfm.cc \ lib/encoders/encoders.cc \ lib/environment.cc \ + lib/fl2.cc \ lib/flags.cc \ lib/fluxmap.cc \ lib/fluxsink/a2rfluxsink.cc \ diff --git a/lib/fl2.cc b/lib/fl2.cc new file mode 100644 index 00000000..bf50430c --- /dev/null +++ b/lib/fl2.cc @@ -0,0 +1,69 @@ +#include "globals.h" +#include "proto.h" +#include "fluxmap.h" +#include "fmt/format.h" +#include "lib/fl2.pb.h" +#include + +static void upgradeFluxFile(FluxFileProto& proto) +{ + if (proto.version() == FluxFileVersion::VERSION_1) + { + /* Change a flux datastream with multiple segments separated by F_DESYNC + * into multiple flux segments. */ + + for (auto& track : *proto.mutable_track()) + { + if (track.flux_size() != 0) + { + Fluxmap oldFlux(track.flux(0)); + + track.clear_flux(); + for (const auto& flux : oldFlux.split()) + track.add_flux(flux->rawBytes()); + } + } + + proto.set_version(FluxFileVersion::VERSION_2); + } + if (proto.version() > FluxFileVersion::VERSION_2) + Error() << fmt::format( + "this is a version {} flux file, but this build of the client can " + "only handle up to version {} --- please upgrade", + proto.version(), + FluxFileVersion::VERSION_2); +} + +FluxFileProto loadFl2File(const std::string filename) +{ + std::ifstream ifs(filename, std::ios::in | std::ios::binary); + if (!ifs.is_open()) + Error() << fmt::format( + "cannot open input file '{}': {}", filename, strerror(errno)); + + char buffer[16]; + ifs.read(buffer, sizeof(buffer)); + if (strncmp(buffer, "SQLite format 3", 16) == 0) + Error() << "this flux file is too old; please use the " + "upgrade-flux-file tool to upgrade it"; + + FluxFileProto proto; + ifs.seekg(0); + if (!proto.ParseFromIstream(&ifs)) + Error() << fmt::format("unable to read input file '{}'", filename); + upgradeFluxFile(proto); + return proto; +} + +void saveFl2File(const std::string filename, FluxFileProto& proto) +{ + proto.set_magic(FluxMagic::MAGIC); + proto.set_version(FluxFileVersion::VERSION_2); + + std::ofstream of(filename, std::ios::out | std::ios::binary); + if (!proto.SerializeToOstream(&of)) + Error() << fmt::format("unable to write output file '{}'", filename); + of.close(); + if (of.fail()) + Error() << "FL2 write I/O error: " << strerror(errno); +} diff --git a/lib/fl2.h b/lib/fl2.h new file mode 100644 index 00000000..0ea4ecfd --- /dev/null +++ b/lib/fl2.h @@ -0,0 +1,10 @@ +#ifndef FL2_H +#define FL2_H + +class FluxFileProto; + +extern FluxFileProto loadFl2File(const std::string filename); +extern void saveFl2File(const std::string filename, FluxFileProto& proto); + +#endif + diff --git a/lib/fluxsink/fl2fluxsink.cc b/lib/fluxsink/fl2fluxsink.cc index 2cb270a3..d6dbbee0 100644 --- a/lib/fluxsink/fl2fluxsink.cc +++ b/lib/fluxsink/fl2fluxsink.cc @@ -9,73 +9,69 @@ #include "proto.h" #include "fmt/format.h" #include "lib/fl2.pb.h" +#include "fl2.h" #include #include #include +#include class Fl2FluxSink : public FluxSink { public: - Fl2FluxSink(const Fl2FluxSinkProto& lconfig): - Fl2FluxSink(lconfig.filename()) - { - } + Fl2FluxSink(const Fl2FluxSinkProto& lconfig): + Fl2FluxSink(lconfig.filename()) + { + } - Fl2FluxSink(const std::string& filename): - _filename(filename), - _of(_filename, std::ios::out | std::ios::binary) - { - if (!_of.is_open()) - Error() << "cannot open output file"; - } + Fl2FluxSink(const std::string& filename): _filename(filename) + { + std::ofstream of(filename); + if (!of.is_open()) + Error() << "cannot open output file"; + of.close(); + std::filesystem::remove(filename); + } - ~Fl2FluxSink() - { - FluxFileProto proto; - proto.set_magic(FluxMagic::MAGIC); - proto.set_version(FluxFileVersion::VERSION_2); - for (const auto& e : _data) - { - auto track = proto.add_track(); - track->set_track(e.first.first); - track->set_head(e.first.second); - for (const auto& fluxBytes : e.second) - track->add_flux(fluxBytes); - } + ~Fl2FluxSink() + { + FluxFileProto proto; + for (const auto& e : _data) + { + auto track = proto.add_track(); + track->set_track(e.first.first); + track->set_head(e.first.second); + for (const auto& fluxBytes : e.second) + track->add_flux(fluxBytes); + } - if (!proto.SerializeToOstream(&_of)) - Error() << "unable to write output file"; - _of.close(); - if (_of.fail()) - Error() << "FL2 write I/O error: " << strerror(errno); - } + saveFl2File(_filename, proto); + } public: - void writeFlux(int track, int head, const Fluxmap& fluxmap) override - { - auto& vector = _data[std::make_pair(track, head)]; - vector.push_back(fluxmap.rawBytes()); - } + void writeFlux(int track, int head, const Fluxmap& fluxmap) override + { + auto& vector = _data[std::make_pair(track, head)]; + vector.push_back(fluxmap.rawBytes()); + } - operator std::string () const override - { - return fmt::format("fl2({})", _filename); - } + operator std::string() const override + { + return fmt::format("fl2({})", _filename); + } private: - std::string _filename; - std::ofstream _of; - std::map, std::vector> _data; + std::string _filename; + std::map, std::vector> _data; }; -std::unique_ptr FluxSink::createFl2FluxSink(const Fl2FluxSinkProto& config) +std::unique_ptr FluxSink::createFl2FluxSink( + const Fl2FluxSinkProto& config) { return std::unique_ptr(new Fl2FluxSink(config)); } -std::unique_ptr FluxSink::createFl2FluxSink(const std::string& filename) +std::unique_ptr FluxSink::createFl2FluxSink( + const std::string& filename) { return std::unique_ptr(new Fl2FluxSink(filename)); } - - diff --git a/lib/fluxsource/fl2fluxsource.cc b/lib/fluxsource/fl2fluxsource.cc index 0785c345..0f21bfeb 100644 --- a/lib/fluxsource/fl2fluxsource.cc +++ b/lib/fluxsource/fl2fluxsource.cc @@ -4,6 +4,7 @@ #include "lib/fl2.pb.h" #include "fluxsource/fluxsource.h" #include "proto.h" +#include "fl2.h" #include "fmt/format.h" #include "fluxmap.h" #include @@ -11,38 +12,36 @@ class Fl2FluxSourceIterator : public FluxSourceIterator { public: - Fl2FluxSourceIterator(const TrackFluxProto& proto): - _proto(proto) - {} + Fl2FluxSourceIterator(const TrackFluxProto& proto): _proto(proto) {} - bool hasNext() const override - { - return _count < _proto.flux_size(); - } + bool hasNext() const override + { + return _count < _proto.flux_size(); + } - std::unique_ptr next() override - { - auto bytes = _proto.flux(_count); - _count++; - return std::make_unique(bytes); - } + std::unique_ptr next() override + { + auto bytes = _proto.flux(_count); + _count++; + return std::make_unique(bytes); + } private: - const TrackFluxProto& _proto; - int _count = 0; + const TrackFluxProto& _proto; + int _count = 0; }; class EmptyFluxSourceIterator : public FluxSourceIterator { - bool hasNext() const override - { - return false; - } + bool hasNext() const override + { + return false; + } - std::unique_ptr next() override - { - Error() << "no flux to read"; - } + std::unique_ptr next() override + { + Error() << "no flux to read"; + } }; class Fl2FluxSource : public FluxSource @@ -50,15 +49,7 @@ class Fl2FluxSource : public FluxSource public: Fl2FluxSource(const Fl2FluxSourceProto& config): _config(config) { - std::ifstream ifs(_config.filename(), std::ios::in | std::ios::binary); - if (!ifs.is_open()) - Error() << fmt::format("cannot open input file '{}': {}", - _config.filename(), - strerror(errno)); - - if (!_proto.ParseFromIstream(&ifs)) - Error() << "unable to read input file"; - upgradeFluxFile(); + _proto = loadFl2File(_config.filename()); } public: @@ -67,7 +58,7 @@ public: for (const auto& trackFlux : _proto.track()) { if ((trackFlux.track() == track) && (trackFlux.head() == head)) - return std::make_unique(trackFlux); + return std::make_unique(trackFlux); } return std::make_unique(); @@ -82,31 +73,6 @@ private: Error() << fmt::format("FL2 read I/O error: {}", strerror(errno)); } - void upgradeFluxFile() - { - if (_proto.version() == FluxFileVersion::VERSION_1) - { - /* Change a flux datastream with multiple segments separated by F_DESYNC into multiple - * flux segments. */ - - for (auto& track : *_proto.mutable_track()) - { - if (track.flux_size() != 0) - { - Fluxmap oldFlux(track.flux(0)); - - track.clear_flux(); - for (const auto& flux : oldFlux.split()) - track.add_flux(flux->rawBytes()); - } - } - - _proto.set_version(FluxFileVersion::VERSION_2); - } - if (_proto.version() > FluxFileVersion::VERSION_2) - Error() << fmt::format("this is a version {} flux file, but this build of the client can only handle up to version {} --- please upgrade", _proto.version(), FluxFileVersion::VERSION_2); - } - private: const Fl2FluxSourceProto& _config; FluxFileProto _proto; @@ -115,12 +81,5 @@ private: std::unique_ptr FluxSource::createFl2FluxSource( const Fl2FluxSourceProto& config) { - char buffer[16]; - std::ifstream(config.filename(), std::ios::in | std::ios::binary) - .read(buffer, 16); - if (strncmp(buffer, "SQLite format 3", 16) == 0) - Error() << "this flux file is too old; please use the " - "upgrade-flux-file tool to upgrade it"; - return std::unique_ptr(new Fl2FluxSource(config)); } diff --git a/src/build.mk b/src/build.mk index 7cc2c7ba..0d3dadd0 100644 --- a/src/build.mk +++ b/src/build.mk @@ -9,6 +9,7 @@ FLUXENGINE_SRCS = \ src/fe-getfileinfo.cc \ src/fe-inspect.cc \ src/fe-ls.cc \ + src/fe-merge.cc \ src/fe-mkdir.cc \ src/fe-mv.cc \ src/fe-rm.cc \ diff --git a/src/fe-merge.cc b/src/fe-merge.cc new file mode 100644 index 00000000..470c10ab --- /dev/null +++ b/src/fe-merge.cc @@ -0,0 +1,65 @@ +#include "globals.h" +#include "flags.h" +#include "fluxmap.h" +#include "sector.h" +#include "proto.h" +#include "flux.h" +#include "fl2.h" +#include "fl2.pb.h" +#include "fmt/format.h" +#include "fluxengine.h" +#include + +static FlagGroup flags; + +static std::vector inputFluxFiles; + +static StringFlag sourceFlux({"-s", "--source"}, + "flux file to read from (repeatable)", + "", + [](const auto& value) + { + inputFluxFiles.push_back(value); + }); + +static StringFlag destFlux( + {"-d", "--dest"}, "destination flux file to write to", ""); + +int mainMerge(int argc, const char* argv[]) +{ + flags.parseFlags(argc, argv); + + if (inputFluxFiles.empty()) + Error() << "you must specify at least one input flux file (with -s)"; + if (destFlux.get() == "") + Error() << "you must specify an output flux file (with -d)"; + + std::map, TrackFluxProto> data; + for (const auto& s : inputFluxFiles) + { + fmt::print("Reading {}...\n", s); + FluxFileProto f = loadFl2File(s); + + for (auto& trackflux : f.track()) + { + auto key = std::make_pair(trackflux.track(), trackflux.head()); + auto i = data.find(key); + if (i == data.end()) + data[key] = trackflux; + else + { + for (auto flux : trackflux.flux()) + i->second.add_flux(flux); + } + } + } + + FluxFileProto proto; + for (auto& i : data) + *proto.add_track() = i.second; + + fmt::print("Writing {}...\n", destFlux.get()); + saveFl2File(destFlux.get(), proto); + + return 0; +} diff --git a/src/fluxengine.cc b/src/fluxengine.cc index cedbdb6a..94166988 100644 --- a/src/fluxengine.cc +++ b/src/fluxengine.cc @@ -12,6 +12,7 @@ extern command_cb mainGetFile; extern command_cb mainGetFileInfo; extern command_cb mainInspect; extern command_cb mainLs; +extern command_cb mainMerge; extern command_cb mainMkDir; extern command_cb mainMv; extern command_cb mainPutFile; @@ -44,6 +45,7 @@ static std::vector commands = { "format", mainFormat, "Format a disk and make a file system on it.", }, { "rawread", mainRawRead, "Reads raw flux from a disk. Warning: you can't use this to copy disks.", }, { "rawwrite", mainRawWrite, "Writes a flux file to a disk. Warning: you can't use this to copy disks.", }, + { "merge", mainMerge, "Merge together multiple flux files.", }, { "getdiskinfo", mainGetDiskInfo, "Read volume metadata off a disk (or image).", }, { "ls", mainLs, "Show files on disk (or image).", }, { "mv", mainMv, "Rename a file on a disk (or image).", },