fsfw_example_public/generators/mod_exporter.py

#! /usr/bin/python3.8
# -*- coding: utf-8 -*-
"""
@file   mod_exporter.py
@brief  Mission Information Base Exporter for the SOURCE project by KSat.
@details
Parses OBSW which is based on FSFW developed by the Institute of Space Systems (IRS) Stuttgart.
Python 3.8 required
This exporter generates the MIB from the SOURCE On-Board Software directly
by using file parser implementations
This exporter has the following capabilities :
    1. Export MIB tables CSV files
    2. Export MIB tables into a SQL database

This exporter currently has parser for following data:
    1. Objects
    2. Returnvalues
    3. Packet content (Telemetry/Telecommands)
    4. Events
    5. Subservices
    6. Device Commands
    7. Global datapool

@developers
Basic Instructions to implement new parserbase:
This parser uses a generic parser class. A specific parser implementation
can be built by implementing the generic parser class.
The parser generally takes a list with all files to parse and a dictionary
with the structure of the MiB table.
This website can be used to experiment with regular expressions: https://regex101.com/

TODO:
    1. Maybe make this file object oriented too.
"""
import os
import pprint

from utility.mib_csv_writer import CsvWriter
from utility.mib_printer import Printer, PrettyPrinter
from utility.mib_sql_writer import SqlWriter
from utility import mib_globals as g
from parserbase.mib_file_list_parser import FileListParser
from packetcontent.mib_packet_content_parser import (
    PacketContentParser,
    PACKET_CONTENT_DEFINITION_DESTINATION,
    PACKET_CONTENT_CSV_NAME,
    PACKET_CONTENT_HEADER_COLUMN,
    SQL_CREATE_PACKET_DATA_CONTENT_CMD,
    SQL_INSERT_PACKET_DATA_CMD,
    SQL_DELETE_PACKET_DATA_CONTENT_CMD
)
from subservice.mib_subservice_parser import (
    SubserviceParser,
    SUBSERVICE_DEFINITION_DESTINATION,
    SUBSERVICE_CSV_NAME,
    SUBSERVICE_COLUMN_HEADER,
    SQL_CREATE_SUBSVC_CMD,
    SQL_DELETE_SUBSVC_CMD,
    SQL_INSERT_INTO_SUBSVC_CMD,
)
from devicecommands.mib_device_command_parser import (
    DeviceHandlerInformationParser,
    DeviceHandlerCommandParser,
    DH_COMMAND_PACKET_DEFINITION_DESTINATION,
    DH_DEFINITION_DESTINATION,
    DH_COMMANDS_CSV_NAME,
    DH_COMMAND_HEADER_COLUMNS,
    SQL_CREATE_CMDTABLE_CMD,
    SQL_INSERT_INTO_CMDTABLE_CMD,
    SQL_DELETE_CMDTABLE_CMD
)
from returnvalues.mib_returnvalues import (
    InterfaceParser,
    ReturnValueParser,
    INTERFACE_DEFINITION_FILES,
    RETURNVALUE_DESTINATIONS,
    sql_retval_exporter,
    CSV_RETVAL_FILENAME
)
from objects.mib_objects import (
    ObjectDefinitionParser,
    OBJECTS_DEFINITIONS,
    export_object_file,
    CSV_OBJECT_FILENAME,
    sql_object_exporter
)
DO_EXPORT_MIB = True
PRINT_TABLES_TO_CONSOLE = False
EXPORT_TO_CSV = True
EXPORT_TO_SQL = True
COPY_FILE = False
COPY_DESTINATION = "."
FILE_SEPARATOR = ";"
EXECUTE_SQL_COMMANDS = False


def main():
    """
    Performs MIB generation.
    """
    parse_mib()


def parse_mib():
    """
    This is the core function. It builds parses all files,
    builds all tables and returns them in a tuple.
    The structure of respective tables is generated in a
    separate functions and is easily modifiable:
    :return:
    """
    handle_subservices_generation()
    print()
    # handle_packet_content_generation()
    # print()
    # handle_device_handler_command_generation()
    # print()
    handle_returnvalue_generation()
    print()
    handle_objects_generation()
    print()
    handle_events_generation()
    print()


def handle_subservices_generation():
    print("MIB Exporter: Parsing subservices")
    subservice_table = generate_subservice_table()
    print("MIB Exporter: Found " + str(len(subservice_table)) + " subservice entries.")
    if PRINT_TABLES_TO_CONSOLE:
        print("MIB Exporter: Printing subservice table: ")
        Printer.print_content(subservice_table)
    if EXPORT_TO_CSV:
        subservice_writer = CsvWriter(
            SUBSERVICE_CSV_NAME, subservice_table, SUBSERVICE_COLUMN_HEADER
        )
        print("MIB Exporter: Exporting to file: " + SUBSERVICE_CSV_NAME)
        subservice_writer.write_to_csv()
    if EXPORT_TO_SQL:
        print("MIB Exporter: Exporting subservices to SQL")
        sql_writer = SqlWriter()
        sql_writer.delete(SQL_DELETE_SUBSVC_CMD)
        sql_writer.sql_writing_helper(
            SQL_CREATE_SUBSVC_CMD, SQL_INSERT_INTO_SUBSVC_CMD, subservice_table
        )


def generate_subservice_table():
    """ Generate the subservice table. """
    subservice_header_parser = FileListParser(
        destination_corrected(SUBSERVICE_DEFINITION_DESTINATION)
    )
    subservice_header_list = subservice_header_parser.parse_header_files(
        False, "MIB Exporter: Parsing subservice files: "
    )
    subservice_file_parser = SubserviceParser(subservice_header_list)
    subservice_table = subservice_file_parser.parse_files()
    return subservice_table


def handle_packet_content_generation():
    print("MIB Exporter: Parsing packing content")
    packet_content_table = generate_packet_content_table()
    print("MIB Exporter: Found " + str(len(packet_content_table)) + " packet content entries.")
    if PRINT_TABLES_TO_CONSOLE:
        print("MIB Exporter: Print packet content table: ")
        Printer.print_content(packet_content_table)
    if EXPORT_TO_CSV:
        packet_content_writer = CsvWriter(
            PACKET_CONTENT_CSV_NAME, packet_content_table, PACKET_CONTENT_HEADER_COLUMN
        )
        print("MIB Exporter: Exporting to file " + PACKET_CONTENT_CSV_NAME)
        packet_content_writer.write_to_csv()
    if EXPORT_TO_SQL:
        print("MIB Exporter: Exporting packet content to SQL")
        sql_writer = SqlWriter()

        sql_writer.sql_writing_helper(
            SQL_CREATE_PACKET_DATA_CONTENT_CMD,
            SQL_INSERT_PACKET_DATA_CMD,
            packet_content_table,
            SQL_DELETE_PACKET_DATA_CONTENT_CMD
        )


def generate_packet_content_table():
    """ Generate packet content table """
    packet_data_header_parser = FileListParser(
        destination_corrected(PACKET_CONTENT_DEFINITION_DESTINATION)
    )
    packet_data_header_list = packet_data_header_parser.parse_header_files(
        False, "MIB Exporter: Parsing packet data files: "
    )
    packet_content_file_parser = PacketContentParser(packet_data_header_list)
    packet_content_table = packet_content_file_parser.parse_files()
    return packet_content_table


def handle_device_handler_command_generation():
    print("MIB Exporter: Parsing device handler commands.")
    dh_command_table = generate_device_command_table()
    print("MIB Exporter: Found " + str(len(dh_command_table)) + " device handler command entries")
    if PRINT_TABLES_TO_CONSOLE:
        print("MIB Exporter: Printing device handler command table: ")
        Printer.print_content(dh_command_table)
    if EXPORT_TO_CSV:
        device_command_writer = CsvWriter(
            DH_COMMANDS_CSV_NAME, dh_command_table, DH_COMMAND_HEADER_COLUMNS
        )
        print("MIB Exporter: Exporting device handler commands to " + DH_COMMANDS_CSV_NAME)
        device_command_writer.write_to_csv()
    if EXPORT_TO_SQL:
        print("MIB Exporter: Exporting device handler commands to SQL")
        sql_writer = SqlWriter()
        sql_writer.sql_writing_helper(
            SQL_CREATE_CMDTABLE_CMD, SQL_INSERT_INTO_CMDTABLE_CMD, dh_command_table,
            SQL_DELETE_CMDTABLE_CMD
        )


def generate_device_command_table(print_info_table: bool = False):
    """ Generate device command table """
    info_header_file_parser = FileListParser(
        destination_corrected(DH_DEFINITION_DESTINATION)
    )
    info_header_file_list = info_header_file_parser.parse_header_files(
        False, "MIB Exporter: Parsing device handler informations: "
    )
    dh_information_parser = DeviceHandlerInformationParser(info_header_file_list)
    dh_information_table = dh_information_parser.parse_files()
    print("MIB Exporter: Found " + str(len(dh_information_table)) +
          " device handler information entries.")
    if print_info_table:
        Printer.print_content(
            dh_information_table, "MIB Exporter: Priting device handler command information table: "
        )

    header_file_parser = FileListParser(
        destination_corrected(DH_COMMAND_PACKET_DEFINITION_DESTINATION)
    )
    header_file_list = header_file_parser.parse_header_files(
        False, "MIB Exporter: Parsing device handler command files: "
    )
    packet_subservice_parser = DeviceHandlerCommandParser(
        header_file_list, dh_information_table
    )
    dh_command_table = packet_subservice_parser.parse_files()
    return dh_command_table


def handle_returnvalue_generation():
    print("MIB Exporter: Parsing returnvalues")
    returnvalue_table = generate_returnvalue_table()
    print("MIB Exporter: Found " + str(len(returnvalue_table)) + " returnvalues.")
    if PRINT_TABLES_TO_CONSOLE:
        print("MIB Exporter: Printing returnvalue table: ")
        Printer.print_content(returnvalue_table)
    if EXPORT_TO_CSV:
        print("MIB Exporter: Exporting returnvalues to " + CSV_RETVAL_FILENAME)
        ReturnValueParser.export_to_file(CSV_RETVAL_FILENAME, returnvalue_table)
    if EXPORT_TO_SQL:
        print("MIB Exporter: Export returnvalues to SQL: ")
        sql_retval_exporter(returnvalue_table)


def generate_returnvalue_table():
    interface_parser = InterfaceParser(
        destination_corrected(INTERFACE_DEFINITION_FILES), False
    )
    interfaces = interface_parser.parse_files()
    print("MIB Exporter: Found interfaces : " + str(len(interfaces)))
    header_parser = FileListParser(destination_corrected(RETURNVALUE_DESTINATIONS))
    header_list = header_parser.parse_header_files(True, "MIB Exporter: Parsing header file list: ")
    returnvalue_parser = ReturnValueParser(interfaces, header_list, False)
    returnvalue_table = returnvalue_parser.parse_files(False)
    if PRINT_TABLES_TO_CONSOLE:
        Printer.print_content(returnvalue_table, "Returnvalue Table: ")
    return returnvalue_table


def handle_objects_generation():
    print("MIB Exporter: Parsing Objects")
    object_parser = ObjectDefinitionParser(destination_corrected(OBJECTS_DEFINITIONS))
    object_table = object_parser.parse_files()
    object_list_sorted = sorted(object_table.items())
    print("MIB Exporter: Found " + str(len(object_table)) + " entries")
    if EXPORT_TO_CSV:
        print("MIB Exporter: Exporting to file: " + CSV_OBJECT_FILENAME)
        export_object_file(CSV_OBJECT_FILENAME, object_list_sorted)
    if EXPORT_TO_SQL:
        print("MIB Exporter: Exporting objects into SQL table")
        sql_object_exporter(object_list_sorted)


def handle_events_generation():
    pass

def destination_corrected(destination_string):
    """
    If headers are parsed here instead of the respective subdirectories,
    the destination files are located in a different relative destination
    """
    if isinstance(destination_string, list):
        destination_list = []
        for destination in destination_string:
            destination_list.append(destination[3:])
        return destination_list

    return destination_string[3:]


def handle_external_file_running():
    """
    Generates the MIB parser from external files
    TODO: Make this stuff OOP too. Retvals and objects were already refactored
    """
    os.chdir("events")
    os.system("python event_parser.py")
    os.chdir("..")
    print_string = "Exported to file: MIB_Events.csv\r\n"
    return print_string


def update_globals():
    """ Updates the global variables """
    g.PP = pprint.PrettyPrinter(indent=0, width=250)
    g.doExportMIB = DO_EXPORT_MIB
    g.executeSQLcommands = False
    g.printToConsole = PRINT_TABLES_TO_CONSOLE
    g.exportToCSV = EXPORT_TO_CSV
    g.copyFile = COPY_FILE
    g.copyDestination = COPY_DESTINATION
    g.fileSeparator = FILE_SEPARATOR


if __name__ == "__main__":
    main()