salfter/fluxengine
1
0
Fork 0
mirror of https://github.com/davidgiven/fluxengine.git synced 2025-10-24 11:11:02 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add the 'fluxengine merge' command.

Browse Source
This commit is contained in:
dg
2023-03-27 20:12:46 +00:00
parent 0a0a72bcf3
commit 81753669cc
9 changed files with 222 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/using.md
View File

@@ -155,6 +155,14 @@ more common tools.
encoding. You can specify a profile if you want to write a subset of the
disk.
- `fluxengine merge -s <fluxfile> -s <fluxfile...> -d <fluxfile`
Merges data from multiple flux files together. This is useful if you have
several reads from an unreliable disk where each read has a different set
of good sectors. By merging the flux files, you get to combine all the
data. Don't use this on reads of different disks, for obvious results! Note
that this works on flux files, not on flux sources.
- `fluxengine inspect -s <flux source> -c <cylinder> -h <head> -B`
Reads flux (possibly from a disk) and does various analyses of it to try

1
lib/build.mk
View File

@@ -9,6 +9,7 @@ LIBFLUXENGINE_SRCS = \
lib/decoders/fmmfm.cc \
lib/encoders/encoders.cc \
lib/environment.cc \
lib/fl2.cc \
lib/flags.cc \
lib/fluxmap.cc \
lib/fluxsink/a2rfluxsink.cc \

69
lib/fl2.cc Normal file
View File

@@ -0,0 +1,69 @@
#include "globals.h"
#include "proto.h"
#include "fluxmap.h"
#include "fmt/format.h"
#include "lib/fl2.pb.h"
#include <fstream>
static void upgradeFluxFile(FluxFileProto& proto)
{
if (proto.version() == FluxFileVersion::VERSION_1)
{
/* Change a flux datastream with multiple segments separated by F_DESYNC
* into multiple flux segments. */
for (auto& track : *proto.mutable_track())
{
if (track.flux_size() != 0)
{
Fluxmap oldFlux(track.flux(0));
track.clear_flux();
for (const auto& flux : oldFlux.split())
track.add_flux(flux->rawBytes());
}
}
proto.set_version(FluxFileVersion::VERSION_2);
}
if (proto.version() > FluxFileVersion::VERSION_2)
Error() << fmt::format(
"this is a version {} flux file, but this build of the client can "
"only handle up to version {} --- please upgrade",
proto.version(),
FluxFileVersion::VERSION_2);
}
FluxFileProto loadFl2File(const std::string filename)
{
std::ifstream ifs(filename, std::ios::in | std::ios::binary);
if (!ifs.is_open())
Error() << fmt::format(
"cannot open input file '{}': {}", filename, strerror(errno));
char buffer[16];
ifs.read(buffer, sizeof(buffer));
if (strncmp(buffer, "SQLite format 3", 16) == 0)
Error() << "this flux file is too old; please use the "
"upgrade-flux-file tool to upgrade it";
FluxFileProto proto;
ifs.seekg(0);
if (!proto.ParseFromIstream(&ifs))
Error() << fmt::format("unable to read input file '{}'", filename);
upgradeFluxFile(proto);
return proto;
}
void saveFl2File(const std::string filename, FluxFileProto& proto)
{
proto.set_magic(FluxMagic::MAGIC);
proto.set_version(FluxFileVersion::VERSION_2);
std::ofstream of(filename, std::ios::out | std::ios::binary);
if (!proto.SerializeToOstream(&of))
Error() << fmt::format("unable to write output file '{}'", filename);
of.close();
if (of.fail())
Error() << "FL2 write I/O error: " << strerror(errno);
}

10
lib/fl2.h Normal file
View File

@@ -0,0 +1,10 @@
#ifndef FL2_H
#define FL2_H
class FluxFileProto;
extern FluxFileProto loadFl2File(const std::string filename);
extern void saveFl2File(const std::string filename, FluxFileProto& proto);
#endif

88
lib/fluxsink/fl2fluxsink.cc
View File

@@ -9,73 +9,69 @@
#include "proto.h"
#include "fmt/format.h"
#include "lib/fl2.pb.h"
#include "fl2.h"
#include <fstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <filesystem>
class Fl2FluxSink : public FluxSink
{
public:
Fl2FluxSink(const Fl2FluxSinkProto& lconfig):
Fl2FluxSink(lconfig.filename())
{
}
Fl2FluxSink(const Fl2FluxSinkProto& lconfig):
Fl2FluxSink(lconfig.filename())
{
}
Fl2FluxSink(const std::string& filename):
_filename(filename),
_of(_filename, std::ios::out | std::ios::binary)
{
if (!_of.is_open())
Error() << "cannot open output file";
}
Fl2FluxSink(const std::string& filename): _filename(filename)
{
std::ofstream of(filename);
if (!of.is_open())
Error() << "cannot open output file";
of.close();
std::filesystem::remove(filename);
}
~Fl2FluxSink()
{
FluxFileProto proto;
proto.set_magic(FluxMagic::MAGIC);
proto.set_version(FluxFileVersion::VERSION_2);
for (const auto& e : _data)
{
auto track = proto.add_track();
track->set_track(e.first.first);
track->set_head(e.first.second);
for (const auto& fluxBytes : e.second)
track->add_flux(fluxBytes);
}
~Fl2FluxSink()
{
FluxFileProto proto;
for (const auto& e : _data)
{
auto track = proto.add_track();
track->set_track(e.first.first);
track->set_head(e.first.second);
for (const auto& fluxBytes : e.second)
track->add_flux(fluxBytes);
}
if (!proto.SerializeToOstream(&_of))
Error() << "unable to write output file";
_of.close();
if (_of.fail())
Error() << "FL2 write I/O error: " << strerror(errno);
}
saveFl2File(_filename, proto);
}
public:
void writeFlux(int track, int head, const Fluxmap& fluxmap) override
{
auto& vector = _data[std::make_pair(track, head)];
vector.push_back(fluxmap.rawBytes());
}
void writeFlux(int track, int head, const Fluxmap& fluxmap) override
{
auto& vector = _data[std::make_pair(track, head)];
vector.push_back(fluxmap.rawBytes());
}
operator std::string () const override
{
return fmt::format("fl2({})", _filename);
}
operator std::string() const override
{
return fmt::format("fl2({})", _filename);
}
private:
std::string _filename;
std::ofstream _of;
std::map<std::pair<unsigned, unsigned>, std::vector<Bytes>> _data;
std::string _filename;
std::map<std::pair<unsigned, unsigned>, std::vector<Bytes>> _data;
};
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(const Fl2FluxSinkProto& config)
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(
const Fl2FluxSinkProto& config)
{
return std::unique_ptr<FluxSink>(new Fl2FluxSink(config));
}
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(const std::string& filename)
std::unique_ptr<FluxSink> FluxSink::createFl2FluxSink(
const std::string& filename)
{
return std::unique_ptr<FluxSink>(new Fl2FluxSink(filename));
}

89
lib/fluxsource/fl2fluxsource.cc
View File

@@ -4,6 +4,7 @@
#include "lib/fl2.pb.h"
#include "fluxsource/fluxsource.h"
#include "proto.h"
#include "fl2.h"
#include "fmt/format.h"
#include "fluxmap.h"
#include <fstream>
@@ -11,38 +12,36 @@
class Fl2FluxSourceIterator : public FluxSourceIterator
{
public:
Fl2FluxSourceIterator(const TrackFluxProto& proto):
_proto(proto)
{}
Fl2FluxSourceIterator(const TrackFluxProto& proto): _proto(proto) {}
bool hasNext() const override
{
return _count < _proto.flux_size();
}
bool hasNext() const override
{
return _count < _proto.flux_size();
}
std::unique_ptr<const Fluxmap> next() override
{
auto bytes = _proto.flux(_count);
_count++;
return std::make_unique<Fluxmap>(bytes);
}
std::unique_ptr<const Fluxmap> next() override
{
auto bytes = _proto.flux(_count);
_count++;
return std::make_unique<Fluxmap>(bytes);
}
private:
const TrackFluxProto& _proto;
int _count = 0;
const TrackFluxProto& _proto;
int _count = 0;
};
class EmptyFluxSourceIterator : public FluxSourceIterator
{
bool hasNext() const override
{
return false;
}
bool hasNext() const override
{
return false;
}
std::unique_ptr<const Fluxmap> next() override
{
Error() << "no flux to read";
}
std::unique_ptr<const Fluxmap> next() override
{
Error() << "no flux to read";
}
};
class Fl2FluxSource : public FluxSource
@@ -50,15 +49,7 @@ class Fl2FluxSource : public FluxSource
public:
Fl2FluxSource(const Fl2FluxSourceProto& config): _config(config)
{
std::ifstream ifs(_config.filename(), std::ios::in | std::ios::binary);
if (!ifs.is_open())
Error() << fmt::format("cannot open input file '{}': {}",
_config.filename(),
strerror(errno));
if (!_proto.ParseFromIstream(&ifs))
Error() << "unable to read input file";
upgradeFluxFile();
_proto = loadFl2File(_config.filename());
}
public:
@@ -67,7 +58,7 @@ public:
for (const auto& trackFlux : _proto.track())
{
if ((trackFlux.track() == track) && (trackFlux.head() == head))
return std::make_unique<Fl2FluxSourceIterator>(trackFlux);
return std::make_unique<Fl2FluxSourceIterator>(trackFlux);
}
return std::make_unique<EmptyFluxSourceIterator>();
@@ -82,31 +73,6 @@ private:
Error() << fmt::format("FL2 read I/O error: {}", strerror(errno));
}
void upgradeFluxFile()
{
if (_proto.version() == FluxFileVersion::VERSION_1)
{
/* Change a flux datastream with multiple segments separated by F_DESYNC into multiple
* flux segments. */
for (auto& track : *_proto.mutable_track())
{
if (track.flux_size() != 0)
{
Fluxmap oldFlux(track.flux(0));
track.clear_flux();
for (const auto& flux : oldFlux.split())
track.add_flux(flux->rawBytes());
}
}
_proto.set_version(FluxFileVersion::VERSION_2);
}
if (_proto.version() > FluxFileVersion::VERSION_2)
Error() << fmt::format("this is a version {} flux file, but this build of the client can only handle up to version {} --- please upgrade", _proto.version(), FluxFileVersion::VERSION_2);
}
private:
const Fl2FluxSourceProto& _config;
FluxFileProto _proto;
@@ -115,12 +81,5 @@ private:
std::unique_ptr<FluxSource> FluxSource::createFl2FluxSource(
const Fl2FluxSourceProto& config)
{
char buffer[16];
std::ifstream(config.filename(), std::ios::in | std::ios::binary)
.read(buffer, 16);
if (strncmp(buffer, "SQLite format 3", 16) == 0)
Error() << "this flux file is too old; please use the "
"upgrade-flux-file tool to upgrade it";
return std::unique_ptr<FluxSource>(new Fl2FluxSource(config));
}

1
src/build.mk
View File

@@ -9,6 +9,7 @@ FLUXENGINE_SRCS = \
src/fe-getfileinfo.cc \
src/fe-inspect.cc \
src/fe-ls.cc \
src/fe-merge.cc \
src/fe-mkdir.cc \
src/fe-mv.cc \
src/fe-rm.cc \

65
src/fe-merge.cc Normal file
View File

@@ -0,0 +1,65 @@
#include "globals.h"
#include "flags.h"
#include "fluxmap.h"
#include "sector.h"
#include "proto.h"
#include "flux.h"
#include "fl2.h"
#include "fl2.pb.h"
#include "fmt/format.h"
#include "fluxengine.h"
#include <fstream>
static FlagGroup flags;
static std::vector<std::string> inputFluxFiles;
static StringFlag sourceFlux({"-s", "--source"},
"flux file to read from (repeatable)",
"",
[](const auto& value)
{
inputFluxFiles.push_back(value);
});
static StringFlag destFlux(
{"-d", "--dest"}, "destination flux file to write to", "");
int mainMerge(int argc, const char* argv[])
{
flags.parseFlags(argc, argv);
if (inputFluxFiles.empty())
Error() << "you must specify at least one input flux file (with -s)";
if (destFlux.get() == "")
Error() << "you must specify an output flux file (with -d)";
std::map<std::pair<int, int>, TrackFluxProto> data;
for (const auto& s : inputFluxFiles)
{
fmt::print("Reading {}...\n", s);
FluxFileProto f = loadFl2File(s);
for (auto& trackflux : f.track())
{
auto key = std::make_pair(trackflux.track(), trackflux.head());
auto i = data.find(key);
if (i == data.end())
data[key] = trackflux;
else
{
for (auto flux : trackflux.flux())
i->second.add_flux(flux);
}
}
}
FluxFileProto proto;
for (auto& i : data)
*proto.add_track() = i.second;
fmt::print("Writing {}...\n", destFlux.get());
saveFl2File(destFlux.get(), proto);
return 0;
}

2
src/fluxengine.cc
View File

@@ -12,6 +12,7 @@ extern command_cb mainGetFile;
extern command_cb mainGetFileInfo;
extern command_cb mainInspect;
extern command_cb mainLs;
extern command_cb mainMerge;
extern command_cb mainMkDir;
extern command_cb mainMv;
extern command_cb mainPutFile;
@@ -44,6 +45,7 @@ static std::vector<Command> commands =
{ "format", mainFormat, "Format a disk and make a file system on it.", },
{ "rawread", mainRawRead, "Reads raw flux from a disk. Warning: you can't use this to copy disks.", },
{ "rawwrite", mainRawWrite, "Writes a flux file to a disk. Warning: you can't use this to copy disks.", },
{ "merge", mainMerge, "Merge together multiple flux files.", },
{ "getdiskinfo", mainGetDiskInfo, "Read volume metadata off a disk (or image).", },
{ "ls", mainLs, "Show files on disk (or image).", },
{ "mv", mainMv, "Rename a file on a disk (or image).", },