Fast Binary Serialization Unity

2024-05-21 17:56 UTC gpt-4o Open in ChatGPT ↗

You

In Unity i have a list of the mocapdata class.

[System.Serializable] public class MocapData { public LimbNames data_header; public List data;

public MocapData() { data = new List(); } }

I want to serialize it (cache it) in the fast way possible, and then load it from disk when the game begins, if a cache exists

ChatGPT

To serialize and deserialize your MocapData list efficiently in Unity, you can use the System.IO and System.Runtime.Serialization.Formatters.Binary namespaces for binary serialization, which is generally faster and produces smaller files compared to text-based serialization like JSON or XML.

Here is a complete example of how you can serialize and deserialize your MocapData list:

1. Define your MocapData and LimbData classes:

using System;
using System.Collections.Generic;

[Serializable]
public class MocapData
{
    public LimbNames data_header;
    public List<LimbData> data;

    public MocapData()
    {
        data = new List<LimbData>();
    }
}

[Serializable]
public class LimbData
{
    // Add your fields and properties here
}

public enum LimbNames
{
    // Define your limb names here
}

2. Serialization and Deserialization methods:

using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using UnityEngine;

public static class MocapDataSerializer
{
    private static string cachePath = Path.Combine(Application.persistentDataPath, "mocapdata.cache");

    public static void Serialize(List<MocapData> mocapDataList)
    {
        BinaryFormatter formatter = new BinaryFormatter();
        using (FileStream stream = new FileStream(cachePath, FileMode.Create))
        {
            formatter.Serialize(stream, mocapDataList);
        }
    }

    public static List<MocapData> Deserialize()
    {
        if (!File.Exists(cachePath))
        {
            return new List<MocapData>();
        }

        BinaryFormatter formatter = new BinaryFormatter();
        using (FileStream stream = new FileStream(cachePath, FileMode.Open))
        {
            return (List<MocapData>)formatter.Deserialize(stream);
        }
    }
}

3. Usage example:

To serialize the list of MocapData:

void SaveMocapData(List<MocapData> mocapDataList)
{
    MocapDataSerializer.Serialize(mocapDataList);
}

To load the cached MocapData when the game starts:

void Start()
{
    List<MocapData> cachedMocapData = MocapDataSerializer.Deserialize();
    if (cachedMocapData != null && cachedMocapData.Count > 0)
    {
        // Use the cached data
    }
    else
    {
        // Load data from another source or initialize it
    }
}

This example demonstrates how to save and load your list of MocapData objects to and from a file on disk using binary serialization. This approach ensures fast serialization and deserialization, making it suitable for caching purposes.

You

use messagepack serializer. here’s my limbdata class

[System.Serializable] public struct LimbData { //public Transform root_data; public int sequence_id; public Matrix4x4 matrix; public Vector3[] positions; public Vector3[] directions; public Vector3 rotation;

/*public LimbData(Transform tr, Vector3[] pos, Vector3[] dir)
{
    //root_data = tr;
    matrix = tr.localToWorldMatrix;
    positions = pos;
    directions = dir;
}*/

public LimbData(int id,Matrix4x4 mat, Vector3 euler, Vector3[] pos, Vector3[] dir )
{
    //root_data = tr;
    sequence_id = id;
    matrix = mat;
    rotation = euler;
    positions = pos.ToArray();
    directions = dir.ToArray();
    //positions = new Vector3[pos.Length];
    //Array.Copy(pos, positions, pos.Length);
    //directions = new Vector3[dir.Length];
    //Array.Copy(dir, directions, dir.Length);
    //positions = pos;
    //directions = dir;
}

public override string ToString()
{
    var sb = new StringBuilder();
    var pos = matrix.GetPosition();
    sb.AppendFormat("{0},{1},{2},{3},{4},{5},",
        Math.Round(rotation.x,3),
        Math.Round(rotation.y,3),
        Math.Round(rotation.z,3),
        Math.Round(pos.x,3),
        Math.Round(pos.y,3),
        Math.Round(pos.z,3));

    var dir_count = directions.Length;

    for (int i = 0; i < dir_count; i++)
    {
        var dir = directions[i];
        sb.AppendFormat("{0},{1},{2},",
            Math.Round(dir.x,3),
            Math.Round(dir.y,3),
            Math.Round(dir.z,3));
    }

    return sb.ToString();

}

}

ChatGPT

To serialize and deserialize your MocapData list using MessagePack in Unity, follow these steps. First, make sure you have the MessagePack package installed. You can add it via the Unity Package Manager by editing your manifest.json:

{
  "dependencies": {
    "com.unity.nuget.newtonsoft-json": "2.0.0",
    "com.cysharp.unitask": "2.3.3",
    "jp.neuecc.messagepack": "2.1.80",
    // other dependencies
  }
}

Or, you can install it via the NuGet package manager if you are using Visual Studio.

Here’s how you can update your MocapData and LimbData classes to work with MessagePack:

1. Define your MocapData and LimbData classes:

using System;
using System.Collections.Generic;
using UnityEngine;
using MessagePack;

[MessagePackObject]
public class MocapData
{
    [Key(0)]
    public LimbNames data_header;
    [Key(1)]
    public List<LimbData> data;

    public MocapData()
    {
        data = new List<LimbData>();
    }
}

[MessagePackObject]
public struct LimbData
{
    [Key(0)]
    public int sequence_id;
    [Key(1)]
    public Matrix4x4 matrix;
    [Key(2)]
    public Vector3[] positions;
    [Key(3)]
    public Vector3[] directions;
    [Key(4)]
    public Vector3 rotation;

    public LimbData(int id, Matrix4x4 mat, Vector3 euler, Vector3[] pos, Vector3[] dir)
    {
        sequence_id = id;
        matrix = mat;
        rotation = euler;
        positions = pos;
        directions = dir;
    }

    public override string ToString()
    {
        var sb = new System.Text.StringBuilder();
        var pos = matrix.GetColumn(3);
        sb.AppendFormat("{0},{1},{2},{3},{4},{5},",
            Math.Round(rotation.x, 3),
            Math.Round(rotation.y, 3),
            Math.Round(rotation.z, 3),
            Math.Round(pos.x, 3),
            Math.Round(pos.y, 3),
            Math.Round(pos.z, 3));

        var dir_count = directions.Length;

        for (int i = 0; i < dir_count; i++)
        {
            var dir = directions[i];
            sb.AppendFormat("{0},{1},{2},",
                Math.Round(dir.x, 3),
                Math.Round(dir.y, 3),
                Math.Round(dir.z, 3));
        }

        return sb.ToString();
    }
}

public enum LimbNames
{
    // Define your limb names here
}

2. Serialization and Deserialization methods:

using System.IO;
using UnityEngine;
using MessagePack;

public static class MocapDataSerializer
{
    private static string cachePath = Path.Combine(Application.persistentDataPath, "mocapdata.cache");

    public static void Serialize(List<MocapData> mocapDataList)
    {
        byte[] data = MessagePackSerializer.Serialize(mocapDataList);
        File.WriteAllBytes(cachePath, data);
    }

    public static List<MocapData> Deserialize()
    {
        if (!File.Exists(cachePath))
        {
            return new List<MocapData>();
        }

        byte[] data = File.ReadAllBytes(cachePath);
        return MessagePackSerializer.Deserialize<List<MocapData>>(data);
    }
}

3. Usage example:

To serialize the list of MocapData:

void SaveMocapData(List<MocapData> mocapDataList)
{
    MocapDataSerializer.Serialize(mocapDataList);
}

To load the cached MocapData when the game starts:

void Start()
{
    List<MocapData> cachedMocapData = MocapDataSerializer.Deserialize();
    if (cachedMocapData != null && cachedMocapData.Count > 0)
    {
        // Use the cached data
    }
    else
    {
        // Load data from another source or initialize it
    }
}

This setup uses MessagePack for Unity to serialize and deserialize your MocapData list efficiently. MessagePack is a binary serialization format that is compact and fast, making it a suitable choice for caching data in Unity.

You

then open the folder where it was saved so i can see it

ChatGPT

To open the folder where the serialized data is saved, you can use Application.OpenURL to open the persistent data path in the file explorer. Here’s how you can modify your script to open the folder after saving the file:

1. Serialization and Deserialization Methods with Folder Opening:

using System.IO;
using UnityEngine;
using MessagePack;

public static class MocapDataSerializer
{
    private static string cachePath = Path.Combine(Application.persistentDataPath, "mocapdata.cache");

    public static void Serialize(List<MocapData> mocapDataList)
    {
        byte[] data = MessagePackSerializer.Serialize(mocapDataList);
        File.WriteAllBytes(cachePath, data);
        OpenCacheFolder();
    }

    public static List<MocapData> Deserialize()
    {
        if (!File.Exists(cachePath))
        {
            return new List<MocapData>();
        }

        byte[] data = File.ReadAllBytes(cachePath);
        return MessagePackSerializer.Deserialize<List<MocapData>>(data);
    }

    public static void OpenCacheFolder()
    {
        string folderPath = Application.persistentDataPath;
#if UNITY_EDITOR
        UnityEditor.EditorUtility.RevealInFinder(folderPath);
#elif UNITY_STANDALONE_WIN
        System.Diagnostics.Process.Start("explorer.exe", folderPath);
#elif UNITY_STANDALONE_OSX
        System.Diagnostics.Process.Start("open", folderPath);
#elif UNITY_STANDALONE_LINUX
        System.Diagnostics.Process.Start("xdg-open", folderPath);
#endif
    }
}

2. Usage example to Save and Open Folder:

To serialize the list of MocapData and open the folder where it was saved:

void SaveMocapData(List<MocapData> mocapDataList)
{
    MocapDataSerializer.Serialize(mocapDataList);
}

To load the cached MocapData when the game starts:

void Start()
{
    List<MocapData> cachedMocapData = MocapDataSerializer.Deserialize();
    if (cachedMocapData != null && cachedMocapData.Count > 0)
    {
        // Use the cached data
    }
    else
    {
        // Load data from another source or initialize it
    }
}

This script includes the OpenCacheFolder method, which uses platform-specific commands to open the file explorer at the location where the cache file is stored. When you call the Serialize method, it will save the data and then open the folder so you can see the saved file.

You

make this function run indeed asynchonously and print in the console as the loop runs

public async Task WriteLimbDataAsText() { Debug.Log(new string(’-’, 80)); Debug.Log(“Writing Data as Text -------”); Debug.Log(new string(’-’, 80));

    var sw = new Stopwatch();
    sw.Start();

    // Get data from Writer
    m_data = writer.mocapData;
    var headers = m_data.data_header.l_names; // Headers: ONLY ONE OCCURENCE
    var data = m_data.data; // Data: PER FRAME OCCURENCE
    var sequence_lengths = m_data.originalSequenceLengths;
    Debug.Log(sequence_lengths.Count);


    var d_count = data.Count;

    // 0 -- CREATE MAIN STRINGBUILDERs ---------
    var sb = new StringBuilder(); // for data
    var l_sb = new StringBuilder(); // for data labels

    // 1 -- WRITE HEADERS ---------------------
    var h_sb = new StringBuilder();
    h_sb.Append("Sequence_ID,");

    for (int i = 0; i < headers.Length; i++)
    {
        if (i == 0)
        {
            h_sb.Append(headers[i] + "_Xpos,");
            h_sb.Append(headers[i] + "_Ypos,");
            h_sb.Append(headers[i] + "_Zpos,");
            h_sb.Append(headers[i] + "_Xrot,");
            h_sb.Append(headers[i] + "_Yrot,");
            h_sb.Append(headers[i] + "_Zrot,");

        }
        else
        {
            h_sb.Append(headers[i] + "_Xdir,");
            h_sb.Append(headers[i] + "_Ydir,");
            h_sb.Append(headers[i] + "_Zdir,");
        }
    }

    sb.AppendLine(h_sb.ToString());
    l_sb.AppendLine("Sequence_ID,Labels");


    /*  var new_lengths = new int[sequence_lengths.Count];

    for (int i = 0; i < sequence_lengths.Count; i++)
     {
         var currentSequenceFrames = sequence_lengths[i];
         var new_length = (currentSequenceFrames / window_length) * window_length;
         Debug.Log($">>> Original frame-count was {currentSequenceFrames} but got converted to {new_length} because of window-size ({window_length})");

         new_lengths[i] = new_length;
     }*/

    window_start_index = 0;
    var step = window_length - window_overlap;
    Debug.Log($"The step is {step} frames");

    // Iterate sequences
    for (int j = 0; j < sequence_lengths.Count; j++)
    {
        // The length of the current sequence
        //var currentSequenceFrames = new_lengths.Length;

        Debug.Log($"--------- SEQUENCE {j} - {sequence_lengths[j]} FRAMES ----------------");

        var window_index = 0;

        while (window_index < sequence_lengths[j] - window_length)
        {
            Debug.Log($">>> JUMP TO {window_index}");
            Debug.Log($">>> >>> local: {window_index} | global:  {window_start_index + window_index}");
            int i = window_index;
            // Grabbing values by full window length
            for ( i = window_index; i < window_index + window_length; i++)
            {

                var index = window_start_index + i;
                //Debug.Log($">>> >>> local: {i} | global:  {index}");

                var d = data[index];
            }

            Debug.Log($">>> >>> local: {i} | global:  {window_start_index + i}");
            // Moving by Step
            window_index += step;
        }

        window_start_index += sequence_lengths[j];
        Debug.Log($"--------- END OF SEQUENCE {j} - GLOBAL INDEX JUMPED TO {window_start_index} ----------------");
    }

ChatGPT

To make your function run asynchronously and print to the console as the loop runs, you can use await Task.Run for the asynchronous execution of the loop and Debug.Log for printing the messages. Here is the updated code:

using System.Diagnostics;
using System.Text;
using System.Threading.Tasks;
using UnityEngine;

public async Task WriteLimbDataAsText()
{
    Debug.Log(new string('-', 80));
    Debug.Log("Writing Data as Text -------");
    Debug.Log(new string('-', 80));

    var sw = new Stopwatch();
    sw.Start();

    // Get data from Writer
    m_data = writer.mocapData;
    var headers = m_data.data_header.l_names; // Headers: ONLY ONE OCCURRENCE
    var data = m_data.data; // Data: PER FRAME OCCURRENCE
    var sequence_lengths = m_data.originalSequenceLengths;
    Debug.Log(sequence_lengths.Count);

    var d_count = data.Count;

    // 0 -- CREATE MAIN STRINGBUILDERs ---------
    var sb = new StringBuilder(); // for data
    var l_sb = new StringBuilder(); // for data labels

    // 1 -- WRITE HEADERS ---------------------
    var h_sb = new StringBuilder();
    h_sb.Append("Sequence_ID,");

    for (int i = 0; i < headers.Length; i++)
    {
        if (i == 0)
        {
            h_sb.Append(headers[i] + "_Xpos,");
            h_sb.Append(headers[i] + "_Ypos,");
            h_sb.Append(headers[i] + "_Zpos,");
            h_sb.Append(headers[i] + "_Xrot,");
            h_sb.Append(headers[i] + "_Yrot,");
            h_sb.Append(headers[i] + "_Zrot,");
        }
        else
        {
            h_sb.Append(headers[i] + "_Xdir,");
            h_sb.Append(headers[i] + "_Ydir,");
            h_sb.Append(headers[i] + "_Zdir,");
        }
    }

    sb.AppendLine(h_sb.ToString());
    l_sb.AppendLine("Sequence_ID,Labels");

    window_start_index = 0;
    var step = window_length - window_overlap;
    Debug.Log($"The step is {step} frames");

    await Task.Run(async () =>
    {
        // Iterate sequences
        for (int j = 0; j < sequence_lengths.Count; j++)
        {
            // The length of the current sequence
            //var currentSequenceFrames = new_lengths.Length;

            Debug.Log($"--------- SEQUENCE {j} - {sequence_lengths[j]} FRAMES ----------------");

            var window_index = 0;

            while (window_index < sequence_lengths[j] - window_length)
            {
                Debug.Log($">>> JUMP TO {window_index}");
                Debug.Log($">>> >>> local: {window_index} | global:  {window_start_index + window_index}");
                int i = window_index;
                // Grabbing values by full window length
                for (i = window_index; i < window_index + window_length; i++)
                {
                    var index = window_start_index + i;
                    //Debug.Log($">>> >>> local: {i} | global:  {index}");

                    var d = data[index];
                    // Simulate a small delay to mimic processing
                    await Task.Yield();
                }

                Debug.Log($">>> >>> local: {i} | global:  {window_start_index + i}");
                // Moving by Step
                window_index += step;
            }

            window_start_index += sequence_lengths[j];
            Debug.Log($"--------- END OF SEQUENCE {j} - GLOBAL INDEX JUMPED TO {window_start_index} ----------------");
        }
    });

    sw.Stop();
    Debug.Log($"Writing Data as Text completed in {sw.ElapsedMilliseconds} ms");
}

Explanation

1. Asynchronous Execution:

   • The core loop logic is wrapped inside an await Task.Run block to run it asynchronously.
   • Within the loop, await Task.Yield() is used to yield control back to the Unity main thread, allowing other operations to continue and keeping the UI responsive.

2. Console Logging:

   • Debug.Log statements are used to print the progress of the loop to the console. These logs will appear in real-time as the loop runs.

3. Simulating Processing Delay:

   • The await Task.Yield() inside the inner loop helps in mimicking a processing delay, making sure the loop doesn’t block the main thread execution.

This approach ensures that your function runs asynchronously and prints progress logs to the console as it processes the data.

You

can i add some constants or variables so that i dont have to repeat values and forget to change stuff?

TRAINING: csv_data_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24.csv csv_label_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24_labels.csv noEpochs: 15 h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5 nNeurons: 15 validation_split: 0.1 batch_size: 10 loss: mse time_step: 45

CLASSIFICATION: trained_h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5 input_file_loc: ./Data/Inference/inference_data.csv threshold: 0

VISUALIZATION: tsne_file_loc: ./Data/Exports/tsne_coordinates_W45_24.csv

MODEL: nFeatures: 31 sequenceLen: 5 label_dict: “1000000”: chair “0100000”: spin “0010000”: ethereal “0001000”: stretch “0000100”: robotic “0000010”: emerge “0000001”: horizonte

ChatGPT

To make your code more maintainable and reduce repetition, you can define constants or variables for commonly used values and file paths. Here’s how you can incorporate constants into your WriteLimbDataAsText method and use a configuration class for other values.

1. Define Constants and Configuration Class:

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEngine;

public static class MocapConstants
{
    public const string CsvDataFileLoc = "./Data/Training/mocap_dataset_2024-05-21_W45_O24.csv";
    public const string CsvLabelFileLoc = "./Data/Training/mocap_dataset_2024-05-21_W45_O24_labels.csv";
    public const int NoEpochs = 15;
    public const string H5FileLoc = "./Data/LSTMWeights/lstm_model_W45_O24.h5";
    public const int NNeurons = 15;
    public const double ValidationSplit = 0.1;
    public const int BatchSize = 10;
    public const string Loss = "mse";
    public const int TimeStep = 45;

    public const string TrainedH5FileLoc = "./Data/LSTMWeights/lstm_model_W45_O24.h5";
    public const string InputFileLoc = "./Data/Inference/inference_data.csv";
    public const int Threshold = 0;

    public const string TsneFileLoc = "./Data/Exports/tsne_coordinates

You

it was .yml

ChatGPT

If you have a .yml configuration file and you want to load these settings into your Unity project, you can use a YAML parser library such as YamlDotNet. Below, I’ll show you how to create a configuration class, load the configuration from the .yml file, and then use these settings in your WriteLimbDataAsText method.

1. Create a Configuration Class:

First, define a class to represent the configuration settings.

using System.Collections.Generic;

public class TrainingConfig
{
    public string CsvDataFileLoc { get; set; }
    public string CsvLabelFileLoc { get; set

You

forget about c#. foxus on yml

ChatGPT

Got it! Here is how you can structure your configuration in a YAML file to avoid repeating values and make it easy to change settings:

YAML Configuration (config.yml)

TRAINING:
  csv_data_file_loc: &csv_data_file_loc ./Data/Training/mocap_dataset_2024-05-21_W45_O24.csv
  csv_label_file_loc: &csv_label_file_loc ./Data/Training/mocap_dataset_2024-05-21_W45_O24_labels.csv
  noEpochs: &noEpochs 15
  h5file_loc: &h5file_loc ./Data/LSTMWeights/lstm_model_W45_O24.h5
  nNeurons: &nNeurons 15
  validation_split: &validation_split 0.1
  batch_size: &batch_size 10
  loss: &loss mse
  time_step: &time_step 45

CLASSIFICATION:
  trained_h5file_loc: *h5file_loc
  input_file_loc: &input_file_loc ./Data/Inference/inference_data.csv
  threshold: &threshold 0

VISUALIZATION:
  tsne_file_loc: &tsne_file_loc ./Data/Exports/tsne_coordinates_W45_24.csv

MODEL:
  nFeatures: &nFeatures 31
  sequenceLen: &sequenceLen 5
  label_dict: &label_dict
    "1000000": chair
    "0100000": spin
    "0010000": ethereal
    "0001000": stretch
    "0000100": robotic
    "0000010": emerge
    "0000001": horizonte

Explanation

• Anchors (&) and Aliases (*): YAML allows you to use anchors (&) to define a value once and then use aliases (*) to reference that value elsewhere in the document. This helps in avoiding repetition and ensures consistency across your configuration.

• Training Section: Contains parameters related to training, such as file locations, number of epochs, etc.

• Classification Section: References the training file location using the alias *h5file_loc.

• Visualization Section: Specifies the location of the TSNE file.

• Model Section: Contains parameters related to the model configuration, including a dictionary for labels.

By structuring your YAML file in this way, you ensure that updates to a commonly used value only need to be made in one place, reducing the risk of errors and making maintenance easier.

You

ok the values that are meant o be repated are 45 and 24

ChatGPT

Understood. Here’s the revised YAML configuration with 45 and 24 values defined once and referenced using anchors and aliases to avoid repetition:

YAML Configuration (config.yml)

defaults:
  window_length: &window_length 45
  window_overlap: &window_overlap 24

TRAINING:
  csv_data_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24.csv
  csv_label_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24_labels.csv
  noEpochs: 15
  h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5
  nNeurons: 15
  validation_split: 0.1
  batch_size: 10
  loss: mse
  time_step: *window_length

CLASSIFICATION:
  trained_h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5
  input_file_loc: ./Data/Inference/inference_data.csv
  threshold: 0

VISUALIZATION:
  tsne_file_loc: ./Data/Exports/tsne_coordinates_W45_24.csv

MODEL:
  nFeatures: 31
  sequenceLen: 5
  label_dict:
    "1000000": chair
    "0100000": spin
    "0010000": ethereal
    "0001000": stretch
    "0000100": robotic
    "0000010": emerge
    "0000001": horizonte
  window_length: *window_length
  window_overlap: *window_overlap

Explanation

1. Defaults Section:

   • window_length: Defined as 45 with anchor &window_length.
   • window_overlap: Defined as 24 with anchor &window_overlap.

2. TRAINING Section:

   • time_step: References the window_length value using the alias *window_length.

3. MODEL Section:

   • window_length and window_overlap: Both reference their respective values using aliases *window_length and *window_overlap.

This approach ensures that if you need to change the values 45 or 24, you only need to update them in one place, reducing the risk of inconsistencies.

You

HOW TO EMBED it into a string

./Data/LSTMWeights/lstm_model_W45_O24.h5

ChatGPT

To embed the values into the strings dynamically within the YAML configuration, you can use a templating approach. While YAML itself does not support templating natively, the idea can be conveyed in a YAML-compatible format, assuming the actual parsing or processing will handle the substitution.

Here’s how you can define the values and then reference them within the strings:

YAML Configuration (config.yml)

defaults:
  window_length: &window_length 45
  window_overlap: &window_overlap 24

TRAINING:
  csv_data_file_loc: !<tag:yaml.org,2002:str> "./Data/Training/mocap_dataset_2024-05-21_W<<window_length>>_O<<window_overlap>>.csv"
  csv_label_file_loc: !<tag:yaml.org,2002:str> "./Data/Training/mocap_dataset_2024-05-21_W<<window_length>>_O<<window_overlap>>_labels.csv"
  noEpochs: 15
  h5file_loc: !<tag:yaml.org,2002:str> "./Data/LSTMWeights/lstm_model_W<<window_length>>_O<<window_overlap>>.h5"
  nNeurons: 15
  validation_split: 0.1
  batch_size: 10
  loss: mse
  time_step: *window_length

CLASSIFICATION:
  trained_h5file_loc: !<tag:yaml.org,2002:str> "./Data/LSTMWeights/lstm_model_W<<window_length>>_O<<window_overlap>>.h5"
  input_file_loc: "./Data/Inference/inference_data.csv"
  threshold: 0

VISUALIZATION:
  tsne_file_loc: !<tag:yaml.org,2002:str> "./Data/Exports/tsne_coordinates_W<<window_length>>_O<<window_overlap>>.csv"

MODEL:
  nFeatures: 31
  sequenceLen: 5
  label_dict:
    "1000000": chair
    "0100000": spin
    "0010000": ethereal
    "0001000": stretch
    "0000100": robotic
    "0000010": emerge
    "0000001": horizonte
  window_length: *window_length
  window_overlap: *window_overlap

Explanation

1. Defaults Section:

   • Define the values for window_length and window_overlap.

2. Training, Classification, Visualization Sections:

   • Use placeholders <<window_length>> and <<window_overlap>> within the strings where the dynamic values should be embedded.

3. !tag:yaml.org,2002:str:

   • YAML tags (e.g., !<tag:yaml.org,2002:str>) indicate that the value should be treated as a string. This can be helpful for processing and replacing placeholders.

Processing the YAML

To replace the placeholders with actual values in your code (e.g., in Python), you would load the YAML, process the strings, and substitute the placeholders.

Here’s an example using Python to read the YAML file, replace placeholders, and print the result:

import yaml

def replace_placeholders(config, values):
    if isinstance(config, dict):
        return {k: replace_placeholders(v, values) for k, v in config.items()}
    elif isinstance(config, list):
        return [replace_placeholders(v, values) for v in config]
    elif isinstance(config, str):
        return config.replace('<<window_length>>', str(values['window_length'])).replace('<<window_overlap>>', str(values['window_overlap']))
    else:
        return config

with open('config.yml', 'r') as file:
    config = yaml.safe_load(file)

values = {
    'window_length': 45,
    'window_overlap': 24
}

processed_config = replace_placeholders(config, values)
print(yaml.dump(processed_config, default_flow_style=False))

Output

This script will output the YAML configuration with the placeholders replaced by the actual values:

CLASSIFICATION:
  input_file_loc: ./Data/Inference/inference_data.csv
  threshold: 0
  trained_h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5
MODEL:
  label_dict:
    '0000001': horizonte
    '0000010': emerge
    '0000100': robotic
    '0001000': stretch
    '0010000': ethereal
    '0100000': spin
    '1000000': chair
  nFeatures: 31
  sequenceLen: 5
  window_length: 45
  window_overlap: 24
TRAINING:
  batch_size: 10
  csv_data_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24.csv
  csv_label_file_loc: ./Data/Training/mocap_dataset_2024-05-21_W45_O24_labels.csv
  h5file_loc: ./Data/LSTMWeights/lstm_model_W45_O24.h5
  loss: mse
  nNeurons: 15
  noEpochs: 15
  time_step: 45
  validation_split: 0.1
VISUALIZATION:
  tsne_file_loc: ./Data/Exports/tsne_coordinates_W45_24.csv
defaults:
  window_length: 45
  window_overlap: 24

This way, you avoid repeating the values directly in the YAML file and make it easier to update them in one place.