salfter/fluxengine
1
0
Fork 0
mirror of https://github.com/davidgiven/fluxengine.git synced 2025-10-31 11:17:01 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add the 'fluxengine merge' command.

Browse Source
This commit is contained in:
dg
2023-03-27 20:12:46 +00:00
parent 0a0a72bcf3
commit 81753669cc
9 changed files with 222 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/using.md
View File

@@ -155,6 +155,14 @@ more common tools.
encoding. You can specify a profile if you want to write a subset of the encoding. You can specify a profile if you want to write a subset of the
disk. disk.
- `fluxengine merge -s <fluxfile> -s <fluxfile...> -d <fluxfile`
Merges data from multiple flux files together. This is useful if you have
several reads from an unreliable disk where each read has a different set
of good sectors. By merging the flux files, you get to combine all the
data. Don't use this on reads of different disks, for obvious results! Note
that this works on flux files, not on flux sources.
- `fluxengine inspect -s <flux source> -c <cylinder> -h <head> -B` - `fluxengine inspect -s <flux source> -c <cylinder> -h <head> -B`
Reads flux (possibly from a disk) and does various analyses of it to try Reads flux (possibly from a disk) and does various analyses of it to try

1
lib/build.mk
View File

@@ -9,6 +9,7 @@ LIBFLUXENGINE_SRCS = \
lib/decoders/fmmfm.cc \ lib/decoders/fmmfm.cc \
lib/encoders/encoders.cc \ lib/encoders/encoders.cc \
lib/environment.cc \ lib/environment.cc \
lib/fl2.cc \
lib/flags.cc \ lib/flags.cc \
lib/fluxmap.cc \ lib/fluxmap.cc \
lib/fluxsink/a2rfluxsink.cc \ lib/fluxsink/a2rfluxsink.cc \

69
lib/fl2.cc Normal file
View File

@@ -0,0 +1,69 @@
#include "globals.h"
#include "proto.h"
#include "fluxmap.h"
#include "fmt/format.h"
#include "lib/fl2.pb.h"
#include <fstream>
static void upgradeFluxFile(FluxFileProto& proto)
{
if (proto.version() == FluxFileVersion::VERSION_1)
{
/* Change a flux datastream with multiple segments separated by F_DESYNC
* into multiple flux segments. */
for (auto& track : *proto.mutable_track())
{
if (track.flux_size() != 0)
{
Fluxmap oldFlux(track.flux(0));
track.clear_flux();
for (const auto& flux : oldFlux.split())
track.add_flux(flux->rawBytes());
}
}
proto.set_version(FluxFileVersion::VERSION_2);
}
if (proto.version() > FluxFileVersion::VERSION_2)
Error() << fmt::format(
"this is a version {} flux file, but this build of the client can "
"only handle up to version {} --- please upgrade",
proto.version(),
FluxFileVersion::VERSION_2);
}
FluxFileProto loadFl2File(const std::string filename)
{
std::ifstream ifs(filename, std::ios::in | std::ios::binary);
if (!ifs.is_open())
Error() << fmt::format(
"cannot open input file '{}': {}", filename, strerror(errno));
char buffer[16];
ifs.read(buffer, sizeof(buffer));
if (strncmp(buffer, "SQLite format 3", 16) == 0)
Error() << "this flux file is too old; please use the "
"upgrade-flux-file tool to upgrade it";
FluxFileProto proto;
ifs.seekg(0);
if (!proto.ParseFromIstream(&ifs))
Error() << fmt::format("unable to read input file '{}'", filename);
upgradeFluxFile(proto);
return proto;
}
void saveFl2File(const std::string filename, FluxFileProto& proto)
{
proto.set_magic(FluxMagic::MAGIC);
proto.set_version(FluxFileVersion::VERSION_2);
std::ofstream of(filename, std::ios::out | std::ios::binary);
if (!proto.SerializeToOstream(&of))
Error() << fmt::format("unable to write output file '{}'", filename);
of.close();
if (of.fail())
Error() << "FL2 write I/O error: " << strerror(errno);
}

10
lib/fl2.h Normal file
View File

@@ -0,0 +1,10 @@
#ifndef FL2_H
#define FL2_H
class FluxFileProto;
extern FluxFileProto loadFl2File(const std::string filename);
extern void saveFl2File(const std::string filename, FluxFileProto& proto);
#endif

30
lib/fluxsink/fl2fluxsink.cc
View File

@@ -9,9 +9,11 @@
#include "proto.h" #include "proto.h"
#include "fmt/format.h" #include "fmt/format.h"
#include "lib/fl2.pb.h" #include "lib/fl2.pb.h"
#include "fl2.h"
#include <fstream> #include <fstream>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <filesystem>
class Fl2FluxSink : public FluxSink class Fl2FluxSink : public FluxSink
{ {
@@ -21,19 +23,18 @@ public:
{ {
} }
Fl2FluxSink(const std::string& filename): Fl2FluxSink(const std::string& filename): _filename(filename)
_filename(filename),
_of(_filename, std::ios::out | std::ios::binary)
{ {
if (!_of.is_open()) std::ofstream of(filename);
if (!of.is_open())
Error() << "cannot open output file"; Error() << "cannot open output file";
of.close();
std::filesystem::remove(filename);
} }
~Fl2FluxSink() ~Fl2FluxSink()
{ {
FluxFileProto proto; FluxFileProto proto;
proto.set_magic(FluxMagic::MAGIC);
proto.set_version(FluxFileVersion::VERSION_2);
for (const auto& e : _data) for (const auto& e : _data)
{ {
auto track = proto.add_track(); auto track = proto.add_track();
@@ -43,11 +44,7 @@ public:
track->add_flux(fluxBytes); track->add_flux(fluxBytes);
} }
if (!proto.SerializeToOstream(&_of)) saveFl2File(_filename, proto);
Error() << "unable to write output file";
_of.close();
if (_of.fail())
Error() << "FL2 write I/O error: " << strerror(errno);
} }
public: public:
@@ -57,25 +54,24 @@ public:
vector.push_back(fluxmap.rawBytes()); vector.push_back(fluxmap.rawBytes());
} }
operator std::string () const override operator std::string() const override
{ {
return fmt::format("fl2({})", _filename); return fmt::format("fl2({})", _filename);
} }
private: private:
std::string _filename; std::string _filename;
std::ofstream _of;
std::map<std::pair<unsigned, unsigned>, std::vector<Bytes>> _data; std::map<std::pair<unsigned, unsigned>, std::vector<Bytes>> _data;
}; };
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(const Fl2FluxSinkProto& config) std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(
const Fl2FluxSinkProto& config)
{ {
return std::unique_ptr<FluxSink>(new Fl2FluxSink(config)); return std::unique_ptr<FluxSink>(new Fl2FluxSink(config));
} }
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(const std::string& filename) std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(
const std::string& filename)
{ {
return std::unique_ptr<FluxSink>(new Fl2FluxSink(filename)); return std::unique_ptr<FluxSink>(new Fl2FluxSink(filename));
} }

47
lib/fluxsource/fl2fluxsource.cc
View File

@@ -4,6 +4,7 @@
#include "lib/fl2.pb.h" #include "lib/fl2.pb.h"
#include "fluxsource/fluxsource.h" #include "fluxsource/fluxsource.h"
#include "proto.h" #include "proto.h"
#include "fl2.h"
#include "fmt/format.h" #include "fmt/format.h"
#include "fluxmap.h" #include "fluxmap.h"
#include <fstream> #include <fstream>
@@ -11,9 +12,7 @@
class Fl2FluxSourceIterator : public FluxSourceIterator class Fl2FluxSourceIterator : public FluxSourceIterator
{ {
public: public:
Fl2FluxSourceIterator(const TrackFluxProto& proto): Fl2FluxSourceIterator(const TrackFluxProto& proto): _proto(proto) {}
_proto(proto)
{}
bool hasNext() const override bool hasNext() const override
{ {
@@ -50,15 +49,7 @@ class Fl2FluxSource : public FluxSource
public: public:
Fl2FluxSource(const Fl2FluxSourceProto& config): _config(config) Fl2FluxSource(const Fl2FluxSourceProto& config): _config(config)
{ {
std::ifstream ifs(_config.filename(), std::ios::in | std::ios::binary); _proto = loadFl2File(_config.filename());
if (!ifs.is_open())
Error() << fmt::format("cannot open input file '{}': {}",
_config.filename(),
strerror(errno));
if (!_proto.ParseFromIstream(&ifs))
Error() << "unable to read input file";
upgradeFluxFile();
} }
public: public:
@@ -82,31 +73,6 @@ private:
Error() << fmt::format("FL2 read I/O error: {}", strerror(errno)); Error() << fmt::format("FL2 read I/O error: {}", strerror(errno));
} }
void upgradeFluxFile()
{
if (_proto.version() == FluxFileVersion::VERSION_1)
{
/* Change a flux datastream with multiple segments separated by F_DESYNC into multiple
* flux segments. */
for (auto& track : *_proto.mutable_track())
{
if (track.flux_size() != 0)
{
Fluxmap oldFlux(track.flux(0));
track.clear_flux();
for (const auto& flux : oldFlux.split())
track.add_flux(flux->rawBytes());
}
}
_proto.set_version(FluxFileVersion::VERSION_2);
}
if (_proto.version() > FluxFileVersion::VERSION_2)
Error() << fmt::format("this is a version {} flux file, but this build of the client can only handle up to version {} --- please upgrade", _proto.version(), FluxFileVersion::VERSION_2);
}
private: private:
const Fl2FluxSourceProto& _config; const Fl2FluxSourceProto& _config;
FluxFileProto _proto; FluxFileProto _proto;
@@ -115,12 +81,5 @@ private:
std::unique_ptr<FluxSource> FluxSource::createFl2FluxSource( std::unique_ptr<FluxSource> FluxSource::createFl2FluxSource(
const Fl2FluxSourceProto& config) const Fl2FluxSourceProto& config)
{ {
char buffer[16];
std::ifstream(config.filename(), std::ios::in | std::ios::binary)
.read(buffer, 16);
if (strncmp(buffer, "SQLite format 3", 16) == 0)
Error() << "this flux file is too old; please use the "
"upgrade-flux-file tool to upgrade it";
return std::unique_ptr<FluxSource>(new Fl2FluxSource(config)); return std::unique_ptr<FluxSource>(new Fl2FluxSource(config));
} }

1
src/build.mk
View File

@@ -9,6 +9,7 @@ FLUXENGINE_SRCS = \
src/fe-getfileinfo.cc \ src/fe-getfileinfo.cc \
src/fe-inspect.cc \ src/fe-inspect.cc \
src/fe-ls.cc \ src/fe-ls.cc \
src/fe-merge.cc \
src/fe-mkdir.cc \ src/fe-mkdir.cc \
src/fe-mv.cc \ src/fe-mv.cc \
src/fe-rm.cc \ src/fe-rm.cc \

65
src/fe-merge.cc Normal file
View File

@@ -0,0 +1,65 @@
#include "globals.h"
#include "flags.h"
#include "fluxmap.h"
#include "sector.h"
#include "proto.h"
#include "flux.h"
#include "fl2.h"
#include "fl2.pb.h"
#include "fmt/format.h"
#include "fluxengine.h"
#include <fstream>
static FlagGroup flags;
static std::vector<std::string> inputFluxFiles;
static StringFlag sourceFlux({"-s", "--source"},
"flux file to read from (repeatable)",
"",
[](const auto& value)
{
inputFluxFiles.push_back(value);
});
static StringFlag destFlux(
{"-d", "--dest"}, "destination flux file to write to", "");
int mainMerge(int argc, const char* argv[])
{
flags.parseFlags(argc, argv);
if (inputFluxFiles.empty())
Error() << "you must specify at least one input flux file (with -s)";
if (destFlux.get() == "")
Error() << "you must specify an output flux file (with -d)";
std::map<std::pair<int, int>, TrackFluxProto> data;
for (const auto& s : inputFluxFiles)
{
fmt::print("Reading {}...\n", s);
FluxFileProto f = loadFl2File(s);
for (auto& trackflux : f.track())
{
auto key = std::make_pair(trackflux.track(), trackflux.head());
auto i = data.find(key);
if (i == data.end())
data[key] = trackflux;
else
{
for (auto flux : trackflux.flux())
i->second.add_flux(flux);
}
}
}
FluxFileProto proto;
for (auto& i : data)
*proto.add_track() = i.second;
fmt::print("Writing {}...\n", destFlux.get());
saveFl2File(destFlux.get(), proto);
return 0;
}

2
src/fluxengine.cc
View File

@@ -12,6 +12,7 @@ extern command_cb mainGetFile;
extern command_cb mainGetFileInfo; extern command_cb mainGetFileInfo;
extern command_cb mainInspect; extern command_cb mainInspect;
extern command_cb mainLs; extern command_cb mainLs;
extern command_cb mainMerge;
extern command_cb mainMkDir; extern command_cb mainMkDir;
extern command_cb mainMv; extern command_cb mainMv;
extern command_cb mainPutFile; extern command_cb mainPutFile;
@@ -44,6 +45,7 @@ static std::vector<Command> commands =
{ "format", mainFormat, "Format a disk and make a file system on it.", }, { "format", mainFormat, "Format a disk and make a file system on it.", },
{ "rawread", mainRawRead, "Reads raw flux from a disk. Warning: you can't use this to copy disks.", }, { "rawread", mainRawRead, "Reads raw flux from a disk. Warning: you can't use this to copy disks.", },
{ "rawwrite", mainRawWrite, "Writes a flux file to a disk. Warning: you can't use this to copy disks.", }, { "rawwrite", mainRawWrite, "Writes a flux file to a disk. Warning: you can't use this to copy disks.", },
{ "merge", mainMerge, "Merge together multiple flux files.", },
{ "getdiskinfo", mainGetDiskInfo, "Read volume metadata off a disk (or image).", }, { "getdiskinfo", mainGetDiskInfo, "Read volume metadata off a disk (or image).", },
{ "ls", mainLs, "Show files on disk (or image).", }, { "ls", mainLs, "Show files on disk (or image).", },
{ "mv", mainMv, "Rename a file on a disk (or image).", }, { "mv", mainMv, "Rename a file on a disk (or image).", },