eive/eive-obsw
eive/eive-obsw
13
4
Fork 0
Code Issues 4 Pull Requests 1 Releases 120 Wiki Activity

FS helpers #604

Merged
muellerr merged 9 commits from core_ctrl_fs_helpers into develop 2023-04-16 02:57:16 +02:00
Conversation 0 Commits 9 Files Changed 5 +431 -47
5 changed files with 431 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CHANGELOG.md
View File

@ -16,6 +16,20 @@ will consitute of a breaking change warranting a new major release:
# [unreleased]
## Added
- Added `mv`, `cp` and `rm` action helpers for the core controller for common filesystem operations.
- Extended directory listing helpers. There is now a directory listing helper which dumps the
directory listing as an action data reply immediately. For smaller directory listings, this
allows a listing without requiring a separate file downlink (which also has not been implemented
yet)
## Changed
- The directory listing action commands now allow compressing of either the target output file
for the directory listing into file action command, or compression in the helper which dumps
the directory listing directly.
# [v1.45.0] 2023-04-14
- q7s-package: v2.5.0

251
bsp_q7s/core/CoreController.cpp
View File

@ -233,6 +233,84 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
case (LIST_DIRECTORY_INTO_FILE): {
return actionListDirectoryIntoFile(actionId, commandedBy, data, size);
}
case (LIST_DIRECTORY_DUMP_DIRECTLY): {
return actionListDirectoryDumpDirectly(actionId, commandedBy, data, size);
}
case (CP_HELPER): {
CpHelperParser parser(data, size);
ReturnValue_t result = parser.parse();
if (result != returnvalue::OK) {
return result;
}
std::ostringstream oss("cp ", std::ostringstream::ate);
if (parser.isRecursiveOptSet()) {
oss << "-r ";
}
auto &sourceTgt = parser.destTgtPair();
oss << sourceTgt.sourceName << " " << sourceTgt.targetName;
sif::info << "CoreController: Performing copy command: " << oss.str() << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
return returnvalue::FAILED;
}
return EXECUTION_FINISHED;
}
case (MV_HELPER): {
MvHelperParser parser(data, size);
ReturnValue_t result = parser.parse();
if (result != returnvalue::OK) {
return result;
}
std::ostringstream oss("mv ", std::ostringstream::ate);
auto &sourceTgt = parser.destTgtPair();
oss << sourceTgt.sourceName << " " << sourceTgt.targetName;
sif::info << "CoreController: Performing move command: " << oss.str() << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
return returnvalue::FAILED;
}
return EXECUTION_FINISHED;
}
case (RM_HELPER): {
RmHelperParser parser(data, size);
ReturnValue_t result = parser.parse();
if (result != returnvalue::OK) {
return result;
}
std::ostringstream oss("rm ", std::ostringstream::ate);
if (parser.isRecursiveOptSet() or parser.isForceOptSet()) {
oss << "-";
}
if (parser.isRecursiveOptSet()) {
oss << "r";
}
if (parser.isForceOptSet()) {
oss << "f";
}
size_t removeTargetSize = 0;
const char *removeTgt = parser.getRemoveTarget(removeTargetSize);
oss << " " << removeTgt;
sif::info << "CoreController: Performing remove command: " << oss.str() << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
return returnvalue::FAILED;
}
return EXECUTION_FINISHED;
}
case (MKDIR_HELPER): {
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
std::string createdDir = std::string(reinterpret_cast<const char *>(data), size);
std::ostringstream oss("mkdir ", std::ostringstream::ate);
oss << createdDir;
sif::info << "CoreController: Performing directory creation: " << oss.str() << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
return returnvalue::FAILED;
}
return EXECUTION_FINISHED;
}
case (SWITCH_REBOOT_FILE_HANDLING): {
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
@ -911,59 +989,144 @@ ReturnValue_t CoreController::initVersionFile() {
return returnvalue::OK;
}
ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId,
ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionId,
MessageQueueId_t commandedBy,
const uint8_t *data, size_t size) {
// TODO: Packet definition for clean deserialization
// 2 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
// null termination, at least 7 bytes for minimum target file name /tmp/a with
// null termination.
if (size < 14) {
return HasActionsIF::INVALID_PARAMETERS;
core::ListDirectoryCmdBase parser(data, size);
ReturnValue_t result = parser.parse();
if (result != returnvalue::OK) {
return result;
}
// We could also make -l optional, but I can't think of a reason why to not use -l..
// This flag specifies to run ls with -a
bool aFlag = data[0];
data += 1;
// This flag specifies to run ls with -R
bool RFlag = data[1];
data += 1;
size_t remainingSize = size - 2;
// One larger for null termination, which prevents undefined behaviour if the sent
// strings are not 0 terminated properly
std::vector<uint8_t> repoAndTargetFileBuffer(remainingSize + 1, 0);
std::memcpy(repoAndTargetFileBuffer.data(), data, remainingSize);
const char *currentCharPtr = reinterpret_cast<const char *>(repoAndTargetFileBuffer.data());
// Full target file name
std::string repoName(currentCharPtr);
size_t repoLength = repoName.length();
// The other string needs to be at least one letter plus NULL termination to be valid at all
// The first string also needs to be NULL terminated, but the termination is not included
// in the string length, so this is subtracted from the remaining size as well
if (repoLength > remainingSize - 3) {
return HasActionsIF::INVALID_PARAMETERS;
}
// The file length will not include the NULL termination, so we skip it
currentCharPtr += repoLength + 1;
std::string targetFileName(currentCharPtr);
std::ostringstream oss;
oss << "ls -l";
if (aFlag) {
std::ostringstream oss("ls -l", std::ostringstream::ate);
if (parser.aFlagSet()) {
oss << "a";
}
if (RFlag) {
if (parser.rFlagSet()) {
oss << "R";
}
oss << " " << repoName << " > " << targetFileName;
int result = std::system(oss.str().c_str());
if (result != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
actionHelper.finish(false, commandedBy, actionId);
size_t repoNameLen = 0;
const char *repoName = parser.getRepoName(repoNameLen);
oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE;
sif::info << "Executing " << oss.str() << " for direct dump";
if (parser.compressionOptionSet()) {
sif::info << " with compression";
}
return returnvalue::OK;
sif::info << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
return returnvalue::FAILED;
}
if (parser.compressionOptionSet()) {
std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".gz");
oss.str("");
oss << "gzip " << LIST_DIR_DUMP_WORK_FILE;
ret = std::system(oss.str().c_str());
if (ret != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
return returnvalue::FAILED;
}
oss.str("");
// Overwrite the work file with the compressed archive.
oss << "mv " << compressedName << " " << LIST_DIR_DUMP_WORK_FILE;
ret = std::system(oss.str().c_str());
if (ret != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
return returnvalue::FAILED;
}
}
std::array<uint8_t, 1024> dirListingBuf{};
dirListingBuf[8] = parser.compressionOptionSet();
// First four bytes reserved for segment index. One byte for compression option information
std::strcpy(reinterpret_cast<char *>(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName);
std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary);
if (ifile.bad()) {
return returnvalue::FAILED;
}
std::error_code e;
size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e);
uint32_t segmentIdx = 0;
size_t dumpedBytes = 0;
size_t nextDumpLen = 0;
size_t dummy = 0;
size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1;
size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1;
uint32_t chunks = totalFileSize / maxDumpLen;
if (totalFileSize % maxDumpLen != 0) {
chunks++;
}
SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy,
dirListingBuf.size() - sizeof(uint32_t),
SerializeIF::Endianness::NETWORK);
while (dumpedBytes < totalFileSize) {
ifile.seekg(dumpedBytes, std::ios::beg);
nextDumpLen = maxDumpLen;
if (totalFileSize - dumpedBytes < maxDumpLen) {
nextDumpLen = totalFileSize - dumpedBytes;
}
SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(),
SerializeIF::Endianness::NETWORK);
ifile.read(reinterpret_cast<char *>(dirListingBuf.data() + listingDataOffset), nextDumpLen);
result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(),
listingDataOffset + nextDumpLen);
if (result != returnvalue::OK) {
// Remove work file when we are done
std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
return result;
}
segmentIdx++;
dumpedBytes += nextDumpLen;
}
// Remove work file when we are done
std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
return EXECUTION_FINISHED;
}
ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId,
MessageQueueId_t commandedBy,
const uint8_t *data, size_t size) {
core::ListDirectoryIntoFile parser(data, size);
ReturnValue_t result = parser.parse();
if (result != returnvalue::OK) {
return result;
}
std::ostringstream oss("ls -l", std::ostringstream::ate);
if (parser.aFlagSet()) {
oss << "a";
}
if (parser.rFlagSet()) {
oss << "R";
}
size_t repoNameLen = 0;
const char *repoName = parser.getRepoName(repoNameLen);
size_t targetFileNameLen = 0;
const char *targetFileName = parser.getTargetName(targetFileNameLen);
oss << " " << repoName << " > " << targetFileName;
sif::info << "Executing list directory request, command: " << oss.str() << std::endl;
int ret = std::system(oss.str().c_str());
if (ret != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
return returnvalue::FAILED;
}
// Compression will add a .gz ending. I don't have any issue with this, it makes it explicit
// that this is a compressed file.
if (parser.compressionOptionSet()) {
oss.str("");
oss << "gzip " << targetFileName;
sif::info << "Compressing directory listing: " << oss.str() << std::endl;
ret = std::system(oss.str().c_str());
if (ret != 0) {
utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
return returnvalue::FAILED;
}
}
return EXECUTION_FINISHED;
}
ReturnValue_t CoreController::initBootCopyFile() {
@ -2049,6 +2212,8 @@ ReturnValue_t CoreController::executeSwUpdate(SwUpdateSources sourceDir, const u
if (not exists(archivePath, e)) {
return HasFileSystemIF::FILE_DOES_NOT_EXIST;
}
// TODO: Decompressing without limiting memory usage with xz is actually a bit risky..
// But has not been an issue so far.
ostringstream cmd("tar -xJf", ios::app);
cmd << " " << archivePath << " -C " << prefixPath;
int result = system(cmd.str().c_str());

7
bsp_q7s/core/CoreController.h
View File

@ -66,6 +66,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs";
static constexpr char CHIP_1_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-nom-rootfs";
static constexpr char CHIP_1_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-gold-rootfs";
static constexpr char LIST_DIR_DUMP_WORK_FILE[] = "/tmp/dir_listing.tmp";
static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000;
static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000;
@ -250,6 +251,12 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy,
const uint8_t* data, size_t size);
ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy,
const uint8_t* data, size_t size);
ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size,
std::ostringstream& oss);
ReturnValue_t actionXscReboot(const uint8_t* data, size_t size);
ReturnValue_t actionReboot(const uint8_t* data, size_t size);

200
mission/sysDefs.h
View File

@ -32,7 +32,6 @@ static constexpr char VERSION_FILE_NAME[] = "version.txt";
static constexpr char REBOOT_FILE_NAME[] = "reboot.txt";
static constexpr char TIME_FILE_NAME[] = "time_backup.txt";
static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0;
static constexpr ActionId_t ANNOUNCE_VERSION = 1;
static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2;
static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3;
@ -59,6 +58,13 @@ static constexpr ActionId_t EXECUTE_SHELL_CMD_BLOCKING = 40;
static constexpr ActionId_t EXECUTE_SHELL_CMD_NON_BLOCKING = 41;
static constexpr ActionId_t SYSTEMCTL_CMD_EXECUTOR = 42;
static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 50;
static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51;
static constexpr ActionId_t CP_HELPER = 52;
static constexpr ActionId_t MV_HELPER = 53;
static constexpr ActionId_t RM_HELPER = 54;
static constexpr ActionId_t MKDIR_HELPER = 55;
static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE;
static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM);
@ -96,6 +102,198 @@ static constexpr Event I2C_REBOOT = event::makeEvent(SUBSYSTEM_ID, 11, severity:
//! [EXPORT] : [COMMENT] PDEC recovery through reset was not possible, performing full reboot.
static constexpr Event PDEC_REBOOT = event::makeEvent(SUBSYSTEM_ID, 12, severity::HIGH);
class ListDirectoryCmdBase {
public: // TODO: Packet definition for clean deserialization
// 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
// null termination
static constexpr size_t MIN_DATA_LEN = 8;
ListDirectoryCmdBase(const uint8_t* data, size_t maxSize) : data(data), maxSize(maxSize) {}
virtual ~ListDirectoryCmdBase() = default;
virtual ReturnValue_t parse() {
if (maxSize < MIN_DATA_LEN) {
return SerializeIF::STREAM_TOO_SHORT;
}
aFlag = data[0];
rFlag = data[1];
compressOption = data[2];
repoNameLen = strnlen(reinterpret_cast<const char*>(data + 3), maxSize - 3);
// Last byte MUST be null terminated!
if (repoNameLen >= maxSize - 3) {
return HasActionsIF::INVALID_PARAMETERS;
}
repoName = reinterpret_cast<const char*>(data + 3);
return returnvalue::OK;
}
bool aFlagSet() const { return this->aFlag; }
bool rFlagSet() const { return this->rFlag; }
bool compressionOptionSet() const { return this->compressOption; }
const char* getRepoName(size_t& repoNameLen) const {
repoNameLen = this->repoNameLen;
return this->repoName;
}
size_t getBaseSize() {
// Include NULL termination
if (repoName != nullptr) {
return 3 + repoNameLen + 1;
}
return 0;
}
protected:
const uint8_t* data;
size_t maxSize;
bool aFlag = false;
bool rFlag = false;
bool compressOption = false;
const char* repoName = nullptr;
size_t repoNameLen = 0;
};
class ListDirectoryIntoFile : public ListDirectoryCmdBase {
public:
// TODO: Packet definition for clean deserialization
// 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
// null termination, at least 7 bytes for minimum target file name /tmp/a with
// null termination.
static constexpr size_t MIN_DATA_LEN = 15;
ListDirectoryIntoFile(const uint8_t* data, size_t maxSize)
: ListDirectoryCmdBase(data, maxSize) {}
ReturnValue_t parse() override {
if (maxSize < MIN_DATA_LEN) {
return SerializeIF::STREAM_TOO_SHORT;
}
ReturnValue_t result = ListDirectoryCmdBase::parse();
if (result != returnvalue::OK) {
return result;
}
targetNameLen =
strnlen(reinterpret_cast<const char*>(data + getBaseSize()), maxSize - getBaseSize());
if (targetNameLen >= maxSize - getBaseSize()) {
// Again: String MUST be null terminated.
return HasActionsIF::INVALID_PARAMETERS;
}
targetName = reinterpret_cast<const char*>(data + getBaseSize());
return result;
}
const char* getTargetName(size_t& targetNameLen) const {
targetNameLen = this->targetNameLen;
return this->targetName;
}
private:
const char* targetName = nullptr;
size_t targetNameLen = 0;
};
struct SourceTargetPair {
const char* sourceName = nullptr;
size_t sourceNameSize = 0;
const char* targetName = nullptr;
size_t targetNameSize = 0;
};
static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen,
SourceTargetPair& destTgt) {
if (maxLen < 4) {
return SerializeIF::STREAM_TOO_SHORT;
}
destTgt.sourceNameSize = strnlen(reinterpret_cast<const char*>(data), maxLen);
if (destTgt.sourceNameSize >= maxLen) {
return HasActionsIF::INVALID_PARAMETERS;
}
destTgt.sourceName = reinterpret_cast<const char*>(data);
size_t remainingLen = maxLen - destTgt.sourceNameSize - 1;
if (remainingLen == 0) {
return HasActionsIF::INVALID_PARAMETERS;
}
destTgt.targetNameSize =
strnlen(reinterpret_cast<const char*>(data + destTgt.sourceNameSize + 1), remainingLen);
if (destTgt.targetNameSize >= remainingLen) {
return HasActionsIF::INVALID_PARAMETERS;
}
destTgt.targetName = reinterpret_cast<const char*>(data + destTgt.sourceNameSize + 1);
return returnvalue::OK;
}
class CpHelperParser {
public:
CpHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
ReturnValue_t parse() {
if (maxLen < 1) {
return SerializeIF::STREAM_TOO_SHORT;
}
recursiveOpt = data[0];
return parseDestTargetString(data + 1, maxLen - 1, destTgt);
}
const SourceTargetPair& destTgtPair() const { return destTgt; }
bool isRecursiveOptSet() const { return recursiveOpt; }
private:
const uint8_t* data;
size_t maxLen;
bool recursiveOpt = false;
SourceTargetPair destTgt;
};
class MvHelperParser {
public:
MvHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
ReturnValue_t parse() { return parseDestTargetString(data, maxLen, destTgt); }
const SourceTargetPair& destTgtPair() const { return destTgt; }
private:
const uint8_t* data;
size_t maxLen;
SourceTargetPair destTgt;
};
class RmHelperParser {
public:
RmHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
ReturnValue_t parse() {
if (maxLen < 2) {
return SerializeIF::STREAM_TOO_SHORT;
}
recursiveOpt = data[0];
forceOpt = data[1];
removeTargetSize = strnlen(reinterpret_cast<const char*>(data + 2), maxLen - 2);
// Must be null-terminated
if (removeTargetSize >= maxLen - 2) {
return HasActionsIF::INVALID_PARAMETERS;
}
removeTarget = reinterpret_cast<const char*>(data + 2);
return returnvalue::OK;
}
bool isRecursiveOptSet() const { return recursiveOpt; }
bool isForceOptSet() const { return forceOpt; }
const char* getRemoveTarget(size_t& removeTargetSize) {
removeTargetSize = this->removeTargetSize;
return removeTarget;
}
private:
const uint8_t* data;
size_t maxLen;
bool recursiveOpt = false;
bool forceOpt = false;
const char* removeTarget = nullptr;
size_t removeTargetSize = 0;
};
} // namespace core
#endif /* MISSION_SYSDEFS_H_ */

2
tmtc

@ -1 +1 @@
Subproject commit d00e4247f66eb2f010f1fe53ee7f59b7fb992481
Subproject commit d623e83be8536f8887f1da00a1f0c4be191db1e8