Data Exporter Tool Overview

2024-10-24 21:32 UTC gpt-4o Open in ChatGPT ↗

You

using System; using System.Diagnostics; using System.IO; using System.Text; using System.Threading.Tasks; using UnityEditor; using UnityEngine; using UnityEngine.Events; using UnityEngine.Serialization; using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport { #if UNITY_EDITOR public class DataExporter2Editor : EditorWindow { [SerializeField] private MocapData_LiteHolder[] mocapDataContainers;

    private int window_length;
    private int window_overlap;
    private string filename;

    private SerializedObject serializedObject;
    private SerializedProperty mocapDataContainersProp;

    [MenuItem("Tools/LSTM - CSV Exporter Tool")]
    public static void ShowWindow()
    {
        GetWindow<DataExporter2Editor>("Data Exporter Tool");
    }

    private void OnEnable()
    {
        serializedObject = new SerializedObject(this);
        mocapDataContainersProp = serializedObject.FindProperty("mocapDataContainers");
    }

    private void OnGUI()
    {
        GUILayout.Label("Export Settings", EditorStyles.boldLabel);

        serializedObject.Update();

        EditorGUILayout.PropertyField(mocapDataContainersProp, new GUIContent("Mocap Data Containers"), true);

        window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
        window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

        GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
        filename = EditorGUILayout.TextField("Filename", filename);

        serializedObject.ApplyModifiedProperties();

        if (GUILayout.Button("Export Data"))
        {
            if (mocapDataContainers == null || mocapDataContainers.Length == 0)
            {
                Debug.LogError("Please assign Mocap Data Containers before exporting.");
                return;
            }
            ExportDataAsync();
        }
    }

    private async void ExportDataAsync()
    {
        string full_name = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
        string path = Path.Combine(Application.streamingAssetsPath, full_name + ".csv");
        string label_path = Path.Combine(Application.streamingAssetsPath, full_name + "_labels.csv");

        Stopwatch masterStopwatch = new Stopwatch();
        masterStopwatch.Start();

        int window_id = 0;
        int frame_counter = 0;
        var data_string = new StringBuilder();
        var label_string = new StringBuilder();

        foreach (var mocapDataHolder in mocapDataContainers)
        {
            var (data, labels) = await BuildFileContent(mocapDataHolder, ref window_id, ref frame_counter);
            data_string.Append(data);
            label_string.Append(labels);
        }

        await WriteToFileAsync(path, data_string.ToString());
        await WriteToFileAsync(label_path, label_string.ToString());

        masterStopwatch.Stop();
        Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

        AssetDatabase.Refresh();
    }

    private async Task<(string, string)> BuildFileContent(MocapData_LiteHolder mocapDataHolder, ref int window_id, ref int frame_counter)
    {
        // Get the mocap data
        MocapData_Lite mocap = mocapDataHolder.mocapData;

        // Find the corresponding AnimationClipList
        AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

        if (clipList == null)
        {
            Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
            return (string.Empty, string.Empty);
        }

        Debug.Log(new string('-', 80));
        Debug.Log($"Writing Data as Text for {mocapDataHolder.name} -------");
        Debug.Log(new string('-', 80));

        var sw = new Stopwatch();
        sw.Start();

        var headers = mocap.data_header;
        var data = mocap.data;
        var sequenceLengths = mocap.framesPerSequence;

        var data_string = new StringBuilder();
        var label_string = new StringBuilder();

        // Append headers only if this is the first call
        if (window_id == 0)
        {
            var header_string = new StringBuilder();
            header_string.Append("Sequence_ID,");

            for (var i = 0; i < headers.Length; i++)
            {
                if (i == 0)
                {
                    header_string.Append(headers[i] + "_Xpos,");
                    header_string.Append(headers[i] + "_Ypos,");
                    header_string.Append(headers[i] + "_Zpos,");
                    header_string.Append(headers[i] + "_Xrot,");
                    header_string.Append(headers[i] + "_Yrot,");
                    header_string.Append(headers[i] + "_Zrot,");
                }
                else
                {
                    header_string.Append(headers[i] + "_Xdir,");
                    header_string.Append(headers[i] + "_Ydir,");
                    header_string.Append(headers[i] + "_Zdir,");
                }
            }

            data_string.AppendLine(header_string.ToString());
            label_string.AppendLine("Sequence_ID,Labels");
        }

        int window_start_index = 0;
        int step = window_length - window_overlap;
        Debug.Log($"The step is {step} frames");

        await Task.Run(async () =>
        {
            for (var j = 0; j < mocap.sequenceCount; j++)
            {
                var seq_length = sequenceLengths[j];
                Debug.Log($">>> SEQUENCE {j} - {seq_length} FRAMES");

                var window_index = 0;

                while (window_index < seq_length - window_length)
                {
                    for (int i = window_index; i < window_index + window_length; i++)
                    {
                        var index = window_start_index + i;
                        var d = data[index];
                        data_string.AppendLine($"{window_id},{d}");
                        label_string.AppendLine($"{window_id},{clipList.classes[d.label_id]}");
                        frame_counter++;

                        await Task.Yield();
                    }

                    window_index += step;
                    window_id++;
                }

                window_start_index += seq_length;
            }
        });

        Debug.Log($">>> Finished writing string data after {sw.Elapsed.TotalSeconds} seconds for {mocapDataHolder.name} -----");
        sw.Stop();

        return (data_string.ToString(), label_string.ToString());
    }

    private AnimationClipList FindClipListForMocapData(MocapData_LiteHolder mocapDataHolder)
    {
        string mocapName = mocapDataHolder.name;
        string[] guids = AssetDatabase.FindAssets($"t:AnimationClipList {mocapName}");

        foreach (string guid in guids)
        {
            string path = AssetDatabase.GUIDToAssetPath(guid);
            AnimationClipList clipList = AssetDatabase.LoadAssetAtPath<AnimationClipList>(path);
            if (clipList != null && clipList.name == mocapName)
            {
                return clipList;
            }
        }

        Debug.LogError($"Could not find AnimationClipList named {mocapName} for MocapData_LiteHolder {mocapName}");
        return null;
    }

    private async Task WriteToFileAsync(string filePath, string content)
    {
        Debug.Log($">>> Saving frames to: {filePath}");
        await using var streamWriter = new StreamWriter(filePath, false);
        await streamWriter.WriteAsync(content);
    }
}
#endif

}

ChatGPT

This script is an Editor Window tool created in Unity for exporting mocap (motion capture) data into a CSV format for use in machine learning models such as LSTM (Long Short-Term Memory) networks. Here’s a breakdown of how the script works and its key features:

Key Components

1. Unity Editor Window Tool: The DataExporter2Editor class extends EditorWindow, which allows it to be displayed as a custom tool inside the Unity editor under the menu item "Tools/LSTM - CSV Exporter Tool". The window provides input fields for users to specify export settings like window length, overlap, and filename.

2. Mocap Data Export: The mocap data is exported as a CSV file where each window (a specified number of frames) of data is extracted, along with its labels. The labels refer to the animation sequences associated with the mocap data, pulled from an AnimationClipList object.

3. GUI for Data Export: The OnGUI() method provides fields for users to input the mocap data containers, set window lengths (in frames), window overlap, and the desired filename for the output CSV. A button (“Export Data”) triggers the export process.

4. Asynchronous Data Export: Data export is handled asynchronously using async/await, improving performance by preventing the editor from freezing during long operations. The method ExportDataAsync() manages the export, creating two CSV files: one for mocap data and another for the associated labels.

5. File Content Generation: The BuildFileContent() method handles the actual data construction, iterating through the mocap data, slicing it into windows, and appending it to a StringBuilder for later writing to a file. The data includes position and rotation information for each frame.

6. Windowing Process: The data is exported in “windows” of a specified length. The tool also allows overlap between windows to ensure better temporal continuity for machine learning tasks. Each window is shifted by window_length - window_overlap.

7. Label Assignment: The method also appends corresponding labels to each sequence of data, which are extracted from an associated AnimationClipList.

8. File Writing: The WriteToFileAsync() method is responsible for writing the constructed data to the CSV files located in the StreamingAssets folder, which is suitable for external data access.

9. Asset Lookup: The FindClipListForMocapData() method looks up AnimationClipList assets in the project that correspond to the mocap data, allowing the tool to link specific mocap sequences to their respective labels.

Potential Usage

• Data Preparation for Machine Learning: This tool is ideal for converting raw mocap data into a CSV format for training machine learning models that require sequence data. By providing control over window sizes and overlap, it allows for the preparation of time-series data suited to LSTM or other sequence-based models.

• Customizable Mocap Export: Users can easily adjust parameters like the length of the window and how much overlap there is between windows, making the tool flexible for different use cases.

Improvements or Customization

1. Error Handling: The script checks for missing data containers and fails gracefully with error messages, but additional checks for null values in the data array might improve robustness.

2. Progress Indication: For large datasets, you could add a progress bar to the Unity Editor window to give feedback on the export progress, improving the user experience.

3. Parallel Processing: While this script already uses asynchronous programming, the actual mocap data extraction could be parallelized across multiple cores if needed, using Task.WhenAll or parallel loops.

4. Additional File Formats: The tool could be extended to export data in other formats, like JSON or binary files, depending on the needs of the specific project.

This script is a well-structured and useful tool for anyone working with motion capture data in Unity, especially in the context of preparing it for machine learning applications.

You

write the code

ChatGPT

Here is the code for the DataExporter2Editor class as per the detailed explanation provided earlier. This Unity Editor script allows for the export of mocap (motion capture) data to CSV format, designed for machine learning purposes (e.g., LSTM models).

using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEditor;
using UnityEngine;
using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport
{
    #if UNITY_EDITOR
    public class DataExporter2Editor : EditorWindow
    {
        [SerializeField]
        private MocapData_LiteHolder[] mocapDataContainers;

        private int window_length;
        private int window_overlap;
        private string filename;

        private SerializedObject serializedObject;
        private SerializedProperty mocapDataContainersProp;

        [MenuItem("Tools/LSTM - CSV Exporter Tool")]
        public static void ShowWindow()
        {
            GetWindow<DataExporter2Editor>("Data Exporter Tool");
        }

        private void OnEnable()
        {
            serializedObject = new SerializedObject(this);
            mocapDataContainersProp = serializedObject.FindProperty("mocapDataContainers");
        }

        private void OnGUI()
        {
            GUILayout.Label("Export Settings", EditorStyles.boldLabel);

            serializedObject.Update();

            EditorGUILayout.PropertyField(mocapDataContainersProp, new GUIContent("Mocap Data Containers"), true);

            window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
            window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

            GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
            filename = EditorGUILayout.TextField("Filename", filename);

            serializedObject.ApplyModifiedProperties();

            if (GUILayout.Button("Export Data"))
            {
                if (mocapDataContainers == null || mocapDataContainers.Length == 0)
                {
                    Debug.LogError("Please assign Mocap Data Containers before exporting.");
                    return;
                }
                ExportDataAsync();
            }
        }

        private async void ExportDataAsync()
        {
            string full_name = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
            string path = Path.Combine(Application.streamingAssetsPath, full_name + ".csv");
            string label_path = Path.Combine(Application.streamingAssetsPath, full_name + "_labels.csv");

            Stopwatch masterStopwatch = new Stopwatch();
            masterStopwatch.Start();

            int window_id = 0;
            int frame_counter = 0;
            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            foreach (var mocapDataHolder in mocapDataContainers)
            {
                var (data, labels) = await BuildFileContent(mocapDataHolder, ref window_id, ref frame_counter);
                data_string.Append(data);
                label_string.Append(labels);
            }

            await WriteToFileAsync(path, data_string.ToString());
            await WriteToFileAsync(label_path, label_string.ToString());

            masterStopwatch.Stop();
            Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

            AssetDatabase.Refresh();
        }

        private async Task<(string, string)> BuildFileContent(MocapData_LiteHolder mocapDataHolder, ref int window_id, ref int frame_counter)
        {
            // Get the mocap data
            MocapData_Lite mocap = mocapDataHolder.mocapData;

            // Find the corresponding AnimationClipList
            AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

            if (clipList == null)
            {
                Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
                return (string.Empty, string.Empty);
            }

            Debug.Log(new string('-', 80));
            Debug.Log($"Writing Data as Text for {mocapDataHolder.name} -------");
            Debug.Log(new string('-', 80));

            var sw = new Stopwatch();
            sw.Start();

            var headers = mocap.data_header;
            var data = mocap.data;
            var sequenceLengths = mocap.framesPerSequence;

            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            // Append headers only if this is the first call
            if (window_id == 0)
            {
                var header_string = new StringBuilder();
                header_string.Append("Sequence_ID,");

                for (var i = 0; i < headers.Length; i++)
                {
                    if (i == 0)
                    {
                        header_string.Append(headers[i] + "_Xpos,");
                        header_string.Append(headers[i] + "_Ypos,");
                        header_string.Append(headers[i] + "_Zpos,");
                        header_string.Append(headers[i] + "_Xrot,");
                        header_string.Append(headers[i] + "_Yrot,");
                        header_string.Append(headers[i] + "_Zrot,");
                    }
                    else
                    {
                        header_string.Append(headers[i] + "_Xdir,");
                        header_string.Append(headers[i] + "_Ydir,");
                        header_string.Append(headers[i] + "_Zdir,");
                    }
                }

                data_string.AppendLine(header_string.ToString());
                label_string.AppendLine("Sequence_ID,Labels");
            }

            int window_start_index = 0;
            int step = window_length - window_overlap;
            Debug.Log($"The step is {step} frames");

            await Task.Run(async () =>
            {
                for (var j = 0; j < mocap.sequenceCount; j++)
                {
                    var seq_length = sequenceLengths[j];
                    Debug.Log($">>> SEQUENCE {j} - {seq_length} FRAMES");

                    var window_index = 0;

                    while (window_index < seq_length - window_length)
                    {
                        for (int i = window_index; i < window_index + window_length; i++)
                        {
                            var index = window_start_index + i;
                            var d = data[index];
                            data_string.AppendLine($"{window_id},{d}");
                            label_string.AppendLine($"{window_id},{clipList.classes[d.label_id]}");
                            frame_counter++;

                            await Task.Yield();
                        }

                        window_index += step;
                        window_id++;
                    }

                    window_start_index += seq_length;
                }
            });

            Debug.Log($">>> Finished writing string data after {sw.Elapsed.TotalSeconds} seconds for {mocapDataHolder.name} -----");
            sw.Stop();

            return (data_string.ToString(), label_string.ToString());
        }

        private AnimationClipList FindClipListForMocapData(MocapData_LiteHolder mocapDataHolder)
        {
            string mocapName = mocapDataHolder.name;
            string[] guids = AssetDatabase.FindAssets($"t:AnimationClipList {mocapName}");

            foreach (string guid in guids)
            {
                string path = AssetDatabase.GUIDToAssetPath(guid);
                AnimationClipList clipList = AssetDatabase.LoadAssetAtPath<AnimationClipList>(path);
                if (clipList != null && clipList.name == mocapName)
                {
                    return clipList;
                }
            }

            Debug.LogError($"Could not find AnimationClipList named {mocapName} for MocapData_LiteHolder {mocapName}");
            return null;
        }

        private async Task WriteToFileAsync(string filePath, string content)
        {
            Debug.Log($">>> Saving frames to: {filePath}");
            await using var streamWriter = new StreamWriter(filePath, false);
            await streamWriter.WriteAsync(content);
        }
    }
    #endif
}

Code Explanation

1. Menu Item: The tool is accessible via Tools > LSTM - CSV Exporter Tool in Unity’s Editor.

2. GUI Layout: The GUI allows users to configure parameters like window_length, window_overlap, and filename, and to assign MocapData_LiteHolder[] objects (which hold mocap data).

3. Export Functionality: When the “Export Data” button is clicked, the script asynchronously processes the mocap data, windowing it as specified, and writes the data to CSV files in the StreamingAssets directory.

4. Mocap Data Handling: Mocap data is windowed using the specified window_length and window_overlap. The motion data and corresponding labels (from AnimationClipList) are extracted and written to separate CSV files.

5. Error Handling: Basic error handling ensures that missing data or animation clip lists are logged to the console.

This script is set up for use in Unity’s editor and will help you export mocap data in a structured format suitable for machine learning tasks.

You

make it accept a folder and get all the mocapdataholders automatically. im bored to drag and drop

ChatGPT

Certainly! To improve the usability and remove the need for dragging and dropping MocapData_LiteHolder objects manually, I’ll modify the script to allow users to select a folder. The script will then automatically retrieve all MocapData_LiteHolder assets within the selected folder.

Here’s the modified version of the script:

Modified DataExporter2Editor to Select Folder Automatically:

using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEditor;
using UnityEngine;
using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport
{
    #if UNITY_EDITOR
    public class DataExporter2Editor : EditorWindow
    {
        private string folderPath = "";
        private int window_length;
        private int window_overlap;
        private string filename;
        private MocapData_LiteHolder[] mocapDataContainers;

        [MenuItem("Tools/LSTM - CSV Exporter Tool")]
        public static void ShowWindow()
        {
            GetWindow<DataExporter2Editor>("Data Exporter Tool");
        }

        private void OnGUI()
        {
            GUILayout.Label("Export Settings", EditorStyles.boldLabel);

            if (GUILayout.Button("Select Mocap Data Folder"))
            {
                folderPath = EditorUtility.OpenFolderPanel("Select Folder Containing MocapData_LiteHolder", "", "");
                if (!string.IsNullOrEmpty(folderPath))
                {
                    LoadMocapDataHoldersFromFolder();
                }
            }

            if (!string.IsNullOrEmpty(folderPath))
            {
                GUILayout.Label($"Selected Folder: {folderPath}", EditorStyles.wordWrappedLabel);
            }

            window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
            window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

            GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
            filename = EditorGUILayout.TextField("Filename", filename);

            if (GUILayout.Button("Export Data"))
            {
                if (mocapDataContainers == null || mocapDataContainers.Length == 0)
                {
                    Debug.LogError("No Mocap Data Containers found in the selected folder.");
                    return;
                }
                ExportDataAsync();
            }
        }

        private void LoadMocapDataHoldersFromFolder()
        {
            if (string.IsNullOrEmpty(folderPath)) return;

            // Convert the system path to a Unity relative path
            string relativeFolderPath = "Assets" + folderPath.Replace(Application.dataPath, "").Replace("\\", "/");

            // Find all MocapData_LiteHolder assets in the folder
            string[] guids = AssetDatabase.FindAssets("t:MocapData_LiteHolder", new[] { relativeFolderPath });
            mocapDataContainers = new MocapData_LiteHolder[guids.Length];

            for (int i = 0; i < guids.Length; i++)
            {
                string assetPath = AssetDatabase.GUIDToAssetPath(guids[i]);
                mocapDataContainers[i] = AssetDatabase.LoadAssetAtPath<MocapData_LiteHolder>(assetPath);
            }

            Debug.Log($"{mocapDataContainers.Length} MocapData_LiteHolder(s) found in folder: {folderPath}");
        }

        private async void ExportDataAsync()
        {
            string full_name = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
            string path = Path.Combine(Application.streamingAssetsPath, full_name + ".csv");
            string label_path = Path.Combine(Application.streamingAssetsPath, full_name + "_labels.csv");

            Stopwatch masterStopwatch = new Stopwatch();
            masterStopwatch.Start();

            int window_id = 0;
            int frame_counter = 0;
            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            foreach (var mocapDataHolder in mocapDataContainers)
            {
                var (data, labels) = await BuildFileContent(mocapDataHolder, ref window_id, ref frame_counter);
                data_string.Append(data);
                label_string.Append(labels);
            }

            await WriteToFileAsync(path, data_string.ToString());
            await WriteToFileAsync(label_path, label_string.ToString());

            masterStopwatch.Stop();
            Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

            AssetDatabase.Refresh();
        }

        private async Task<(string, string)> BuildFileContent(MocapData_LiteHolder mocapDataHolder, ref int window_id, ref int frame_counter)
        {
            MocapData_Lite mocap = mocapDataHolder.mocapData;
            AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

            if (clipList == null)
            {
                Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
                return (string.Empty, string.Empty);
            }

            var sw = new Stopwatch();
            sw.Start();

            var headers = mocap.data_header;
            var data = mocap.data;
            var sequenceLengths = mocap.framesPerSequence;

            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            if (window_id == 0)
            {
                var header_string = new StringBuilder();
                header_string.Append("Sequence_ID,");

                for (var i = 0; i < headers.Length; i++)
                {
                    if (i == 0)
                    {
                        header_string.Append(headers[i] + "_Xpos,");
                        header_string.Append(headers[i] + "_Ypos,");
                        header_string.Append(headers[i] + "_Zpos,");
                        header_string.Append(headers[i] + "_Xrot,");
                        header_string.Append(headers[i] + "_Yrot,");
                        header_string.Append(headers[i] + "_Zrot,");
                    }
                    else
                    {
                        header_string.Append(headers[i] + "_Xdir,");
                        header_string.Append(headers[i] + "_Ydir,");
                        header_string.Append(headers[i] + "_Zdir,");
                    }
                }

                data_string.AppendLine(header_string.ToString());
                label_string.AppendLine("Sequence_ID,Labels");
            }

            int window_start_index = 0;
            int step = window_length - window_overlap;

            await Task.Run(async () =>
            {
                for (var j = 0; j < mocap.sequenceCount; j++)
                {
                    var seq_length = sequenceLengths[j];
                    var window_index = 0;

                    while (window_index < seq_length - window_length)
                    {
                        for (int i = window_index; i < window_index + window_length; i++)
                        {
                            var index = window_start_index + i;
                            var d = data[index];
                            data_string.AppendLine($"{window_id},{d}");
                            label_string.AppendLine($"{window_id},{clipList.classes[d.label_id]}");
                            frame_counter++;

                            await Task.Yield();
                        }

                        window_index += step;
                        window_id++;
                    }

                    window_start_index += seq_length;
                }
            });

            sw.Stop();

            return (data_string.ToString(), label_string.ToString());
        }

        private AnimationClipGot it! If you'd like to select a folder directly from Unity's **Assets** folder rather than using the system file explorer, we can use Unity's `EditorUtility.OpenFolderPanel` to select a folder within the Unity project and then convert that folder path into a relative Unity path. Here's the updated script to allow folder selection directly from the **Assets** folder:

### Modified Code to Select Folder in Assets:

```csharp
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEditor;
using UnityEngine;
using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport
{
    #if UNITY_EDITOR
    public class DataExporter2Editor : EditorWindow
    {
        private string folderPath = "";
        private int window_length;
        private int window_overlap;
        private string filename;
        private MocapData_LiteHolder[] mocapDataContainers;

        [MenuItem("Tools/LSTM - CSV Exporter Tool")]
        public static void ShowWindow()
        {
            GetWindow<DataExporter2Editor>("Data Exporter Tool");
        }

        private void OnGUI()
        {
            GUILayout.Label("Export Settings", EditorStyles.boldLabel);

            if (GUILayout.Button("Select Mocap Data Folder in Assets"))
            {
                string selectedFolder = EditorUtility.OpenFolderPanel("Select Folder Containing MocapData_LiteHolder", "Assets", "");

                if (!string.IsNullOrEmpty(selectedFolder))
                {
                    folderPath = GetRelativeUnityPath(selectedFolder);
                    LoadMocapDataHoldersFromFolder();
                }
            }

            if (!string.IsNullOrEmpty(folderPath))
            {
                GUILayout.Label($"Selected Folder: {folderPath}", EditorStyles.wordWrappedLabel);
            }

            window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
            window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

            GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
            filename = EditorGUILayout.TextField("Filename", filename);

            if (GUILayout.Button("Export Data"))
            {
                if (mocapDataContainers == null || mocapDataContainers.Length == 0)
                {
                    Debug.LogError("No Mocap Data Containers found in the selected folder.");
                    return;
                }
                ExportDataAsync();
            }
        }

        private void LoadMocapDataHoldersFromFolder()
        {
            if (string.IsNullOrEmpty(folderPath)) return;

            // Find all MocapData_LiteHolder assets in the folder
            string[] guids = AssetDatabase.FindAssets("t:MocapData_LiteHolder", new[] { folderPath });
            mocapDataContainers = new MocapData_LiteHolder[guids.Length];

            for (int i = 0; i < guids.Length; i++)
            {
                string assetPath = AssetDatabase.GUIDToAssetPath(guids[i]);
                mocapDataContainers[i] = AssetDatabase.LoadAssetAtPath<MocapData_LiteHolder>(assetPath);
            }

            Debug.Log($"{mocapDataContainers.Length} MocapData_LiteHolder(s) found in folder: {folderPath}");
        }

        private async void ExportDataAsync()
        {
            string full_name = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
            string path = Path.Combine(Application.streamingAssetsPath, full_name + ".csv");
            string label_path = Path.Combine(Application.streamingAssetsPath, full_name + "_labels.csv");

            Stopwatch masterStopwatch = new Stopwatch();
            masterStopwatch.Start();

            int window_id = 0;
            int frame_counter = 0;
            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            foreach (var mocapDataHolder in mocapDataContainers)
            {
                var (data, labels) = await BuildFileContent(mocapDataHolder, ref window_id, ref frame_counter);
                data_string.Append(data);
                label_string.Append(labels);
            }

            await WriteToFileAsync(path, data_string.ToString());
            await WriteToFileAsync(label_path, label_string.ToString());

            masterStopwatch.Stop();
            Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

            AssetDatabase.Refresh();
        }

        private async Task<(string, string)> BuildFileContent(MocapData_LiteHolder mocapDataHolder, ref int window_id, ref int frame_counter)
        {
            MocapData_Lite mocap = mocapDataHolder.mocapData;
            AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

            if (clipList == null)
            {
                Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
                return (string.Empty, string.Empty);
            }

            var sw = new Stopwatch();
            sw.Start();

            var headers = mocap.data_header;
            var data = mocap.data;
            var sequenceLengths = mocap.framesPerSequence;

            var data_string = new StringBuilder();
            var label_string = new StringBuilder();

            if (window_id == 0)
            {
                var header_string = new StringBuilder();
                header_string.Append("Sequence_ID,");

                for (var i = 0; i < headers.Length; i++)
                {
                    if (i == 0)
                    {
                        header_string.Append(headers[i] + "_Xpos,");
                        header_string.Append(headers[i] + "_Ypos,");
                        header_string.Append(headers[i] + "_Zpos,");
                        header_string.Append(headers[i] + "_Xrot,");
                        header_string.Append(headers[i] + "_Yrot,");
                        header_string.Append(headers[i] + "_Zrot,");
                    }
                    else
                    {
                        header_string.Append(headers[i] + "_Xdir,");
                        header_string.Append(headers[i] + "_Ydir,");
                        header_string.Append(headers[i] + "_Zdir,");
                    }
                }

                data_string.AppendLine(header_string.ToString());
                label_string.AppendLine("Sequence_ID,Labels");
            }

            int window_start_index = 0;
            int step = window_length - window_overlap;

            await Task.Run(async () =>
            {
                for (var j = 0; j < mocap.sequenceCount; j++)
                {
                    var seq_length = sequenceLengths[j];
                    var window_index = 0;

                    while (window_index < seq_length - window_length)
                    {
                        for (int i = window_index; i < window_index + window_length; i++)
                        {
                            var index = window_start_index + i;
                            var d = data[index];
                            data_string.AppendLine($"{window_id},{d}");
                            label_string.AppendLine($"{window_id},{clipList.classes[d.label_id]}");
                            frame_counter++;

                            await Task.Yield();
                        }

                        window_index += step;
                        window_id++;
                    }

                    window_start_index += seq_length;
                }
            });

            sw.Stop();

            return (data_string.ToString(), label_string.ToString());
        }

        private AnimationClipList FindClipListForMocapData(MocapData_LiteHolder mocapDataHolder)
        {
            string mocapName = mocapDataHolder.name;
            string[] guids = AssetDatabase.FindAssets($"t:AnimationClipList {mocapName}");

            foreach (string guid in guids)
            {
                string path = AssetDatabase.GUIDToAssetPath(guid);
                AnimationClipList clipList = AssetDatabase.LoadAssetAtPath<AnimationClipList>(path);
                if (clipList != null && clipList.name == mocapName)
                {
                    return clipList;
                }
            }

            Debug.LogError($"Could not find AnimationClipList named {mocapName} for MocapData_LiteHolder {mocapName}");
            return null;
        }

        private async Task WriteToFileAsync(string filePath, string content)
        {
            Debug.Log($">>> Saving frames to: {filePath}");
            await using var streamWriter = new StreamWriter(filePath, false);
            await streamWriter.WriteAsync(content);
        }

        private string GetRelativeUnityPath(string fullPath)
        {
            if (fullPath.StartsWith(Application.dataPath))
            {
                return "Assets" + fullPath.Substring(Application.dataPath.Length);
            }

            return fullPath;
        }
    }
    #endif
}

Key Changes:

1. Folder Selection Inside Unity’s Assets:

   • The folder is selected using EditorUtility.OpenFolderPanel(). After selecting the folder, the full system path is converted into a Unity-relative path using the GetRelativeUnityPath() method.

2. Automatic Mocap Data Loading:

   • After selecting the folder, the script automatically loads all MocapData_LiteHolder assets in the selected folder using AssetDatabase.FindAssets() and stores them in the mocapDataContainers array for export.

3. Path Conversion:

   • The GetRelativeUnityPath() method converts the selected folder’s system path to a relative path (from “Assets”), ensuring Unity can work with it using AssetDatabase.

Workflow:

1. Select Folder in Assets:
   • When you click the “Select Mocap Data Folder in Assets” button, you can select any folder within your Unity project’s Assets directory.
2. Load Mocap Data Holders:
   • Once a folder is selected, all MocapData_LiteHolder assets in that folder are automatically loaded for processing.
3. Export Data:
   • The data is processed as described before, generating two CSV files (one for the data and another for the labels) based on the specified window and

You

as you see the data are organized in windows. in the end after LSTM training, we have a latent space of ORIGINAL DATA COUNT / WINDOW SIZE.

For each point, we must be able to look-up and find the original data sequence.

As we are processing the colelction of data in this editor script, we must build a look-up table, where each id of the LSTM data, corresponds to: (animationcliplist_id, clip_id, start frame, end frame)

ChatGPT

To accomplish this, I’ll extend the existing DataExporter2Editor script to include a lookup table that maps each generated window ID to the original data sequence. This table will store information that uniquely identifies each window by tracking the (animationcliplist_id, clip_id, start frame, end frame) for each window.

Here’s how the new lookup table will be handled:

1. Create a Lookup Table CSV: As each window is processed, a lookup entry will be created with the specified information, including the animationcliplist_id, clip_id, and start and end frames.

2. Export the Lookup Table: At the end of the process, a new CSV file (e.g., lookup_table.csv) will be saved alongside the data and label CSVs, containing the lookup information for each window.

Updated Code

Below is the modified code that includes building and exporting this lookup table:

using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEditor;
using UnityEngine;
using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport
{
    #if UNITY_EDITOR
    public class DataExporter2Editor : EditorWindow
    {
        private string folderPath = "";
        private int window_length;
        private int window_overlap;
        private string filename;
        private MocapData_LiteHolder[] mocapDataContainers;

        [MenuItem("Tools/LSTM - CSV Exporter Tool")]
        public static void ShowWindow()
        {
            GetWindow<DataExporter2Editor>("Data Exporter Tool");
        }

        private void OnGUI()
        {
            GUILayout.Label("Export Settings", EditorStyles.boldLabel);

            if (GUILayout.Button("Select Mocap Data Folder in Assets"))
            {
                string selectedFolder = EditorUtility.OpenFolderPanel("Select Folder Containing MocapData_LiteHolder", "Assets", "");

                if (!string.IsNullOrEmpty(selectedFolder))
                {
                    folderPath = GetRelativeUnityPath(selectedFolder);
                    LoadMocapDataHoldersFromFolder();
                }
            }

            if (!string.IsNullOrEmpty(folderPath))
            {
                GUILayout.Label($"Selected Folder: {folderPath}", EditorStyles.wordWrappedLabel);
            }

            window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
            window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

            GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
            filename = EditorGUILayout.TextField("Filename", filename);

            if (GUILayout.Button("Export Data"))
            {
                if (mocapDataContainers == null || mocapDataContainers.Length == 0)
                {
                    Debug.LogError("No Mocap Data Containers found in the selected folder.");
                    return;
                }
                ExportDataAsync();
            }
        }

        private void LoadMocapDataHoldersFromFolder()
        {
            if (string.IsNullOrEmpty(folderPath)) return;

            string[] guids = AssetDatabase.FindAssets("t:MocapData_LiteHolder", new[] { folderPath });
            mocapDataContainers = new MocapData_LiteHolder[guids.Length];

            for (int i = 0; i < guids.Length; i++)
            {
                string assetPath = AssetDatabase.GUIDToAssetPath(guids[i]);
                mocapDataContainers[i] = AssetDatabase.LoadAssetAtPath<MocapData_LiteHolder>(assetPath);
            }

            Debug.Log($"{mocapDataContainers.Length} MocapData_LiteHolder(s) found in folder: {folderPath}");
        }

        private async void ExportDataAsync()
        {
            string baseFileName = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
            string path = Path.Combine(Application.streamingAssetsPath, baseFileName + ".csv");
            string labelPath = Path.Combine(Application.streamingAssetsPath, baseFileName + "_labels.csv");
            string lookupPath = Path.Combine(Application.streamingAssetsPath, baseFileName + "_lookup_table.csv");

            Stopwatch masterStopwatch = new Stopwatch();
            masterStopwatch.Start();

            int windowId = 0;
            int frameCounter = 0;
            var dataString = new StringBuilder();
            var labelString = new StringBuilder();
            var lookupString = new StringBuilder("Window_ID,AnimationClipList_ID,Clip_ID,Start_Frame,End_Frame\n");

            foreach (var mocapDataHolder in mocapDataContainers)
            {
                var (data, labels, lookupEntries) = await BuildFileContent(mocapDataHolder, ref windowId, ref frameCounter);
                dataString.Append(data);
                labelString.Append(labels);
                lookupString.Append(lookupEntries);
            }

            await WriteToFileAsync(path, dataString.ToString());
            await WriteToFileAsync(labelPath, labelString.ToString());
            await WriteToFileAsync(lookupPath, lookupString.ToString());

            masterStopwatch.Stop();
            Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

            AssetDatabase.Refresh();
        }

        private async Task<(string, string, string)> BuildFileContent(MocapData_LiteHolder mocapDataHolder, ref int windowId, ref int frameCounter)
        {
            MocapData_Lite mocap = mocapDataHolder.mocapData;
            AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

            if (clipList == null)
            {
                Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
                return (string.Empty, string.Empty, string.Empty);
            }

            var sw = new Stopwatch();
            sw.Start();

            var headers = mocap.data_header;
            var data = mocap.data;
            var sequenceLengths = mocap.framesPerSequence;

            var dataString = new StringBuilder();
            var labelString = new StringBuilder();
            var lookupString = new StringBuilder();

            if (windowId == 0)
            {
                var headerString = new StringBuilder("Sequence_ID,");

                for (var i = 0; i < headers.Length; i++)
                {
                    if (i == 0)
                    {
                        headerString.Append($"{headers[i]}_Xpos,{headers[i]}_Ypos,{headers[i]}_Zpos,{headers[i]}_Xrot,{headers[i]}_Yrot,{headers[i]}_Zrot,");
                    }
                    else
                    {
                        headerString.Append($"{headers[i]}_Xdir,{headers[i]}_Ydir,{headers[i]}_Zdir,");
                    }
                }

                dataString.AppendLine(headerString.ToString());
                labelString.AppendLine("Sequence_ID,Labels");
            }

            int windowStartIndex = 0;
            int step = window_length - window_overlap;

            await Task.Run(async () =>
            {
                for (var j = 0; j < mocap.sequenceCount; j++)
                {
                    var seqLength = sequenceLengths[j];
                    var clipId = j; // Assuming each sequence represents a clip
                    var windowIndex = 0;

                    while (windowIndex < seqLength - window_length)
                    {
                        int startFrame = windowStartIndex + windowIndex;
                        int endFrame = startFrame + window_length - 1;

                        for (int i = windowIndex; i < windowIndex + window_length; i++)
                        {
                            var index = windowStartIndex + i;
                            var d = data[index];
                            dataString.AppendLine($"{windowId},{d}");
                            labelString.AppendLine($"{windowId},{clipList.classes[d.label_id]}");
                            frameCounter++;

                            await Task.Yield();
                        }

                        lookupString.AppendLine($"{windowId},{mocapDataHolder.name},{clipId},{startFrame},{endFrame}");
                        windowIndex += step;
                        windowId++;
                    }

                    windowStartIndex += seqLength;
                }
            });

            sw.Stop();
            return (dataString.ToString(), labelString.ToString(), lookupString.ToString());
        }

        private AnimationClipList FindClipListForMocapData(MocapData_LiteHolder mocapDataHolder)
        {
            string mocapName = mocapDataHolder.name;
            string[] guids = AssetDatabase.FindAssets($"t:AnimationClipList {mocapName}");

            foreach (string guid in guids)
            {
                string path = AssetDatabase.GUIDToAssetPath(guid);
                AnimationClipList clipList = AssetDatabase.LoadAssetAtPath<AnimationClipList>(path);
                if (clipList != null && clipList.name == mocapName)
                {
                    return clipList;
                }
            }

            Debug.LogError($"Could not find AnimationClipList named {mocapName} for MocapData_LiteHolder {mocapName}");
            return null;
        }

        private async Task WriteToFileAsync(string filePath, string content)
        {
            Debug.Log($">>> Saving frames to: {filePath}");
            await using var streamWriter = new StreamWriter(filePath, false);
            await streamWriter.WriteAsync(content);
        }

        private string GetRelativeUnityPath(string fullPath)
        {
            if (fullPath.StartsWith(Application.dataPath))
            {
                return "Assets" + fullPath.Substring(Application.dataPath.Length);
            }

            return fullPath;
        }
    }
    #endif
}

Explanation of Changes

1. Lookup Table Generation:
   • Each window is processed and an entry is added to lookupString with the windowId, mocapDataHolder.name (as animationcliplist_id), clipId, startFrame, and endFrame.

2. Export Lookup Table CSV:

You

we cant have ref in a Task

ChatGPT

You’re correct! Using ref parameters in async Task methods causes issues since ref values can’t be safely used across asynchronous boundaries. We can adjust the design by avoiding ref parameters in the asynchronous task and instead updating windowId and frameCounter after the task completes.

Here’s the modified version without ref parameters:

using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using UnityEditor;
using UnityEngine;
using Debug = UnityEngine.Debug;

namespace UnchartedLimbo.Core.Utilities.DataExport
{
    #if UNITY_EDITOR
    public class DataExporter2Editor : EditorWindow
    {
        private string folderPath = "";
        private int window_length;
        private int window_overlap;
        private string filename;
        private MocapData_LiteHolder[] mocapDataContainers;

        [MenuItem("Tools/LSTM - CSV Exporter Tool")]
        public static void ShowWindow()
        {
            GetWindow<DataExporter2Editor>("Data Exporter Tool");
        }

        private void OnGUI()
        {
            GUILayout.Label("Export Settings", EditorStyles.boldLabel);

            if (GUILayout.Button("Select Mocap Data Folder in Assets"))
            {
                string selectedFolder = EditorUtility.OpenFolderPanel("Select Folder Containing MocapData_LiteHolder", "Assets", "");

                if (!string.IsNullOrEmpty(selectedFolder))
                {
                    folderPath = GetRelativeUnityPath(selectedFolder);
                    LoadMocapDataHoldersFromFolder();
                }
            }

            if (!string.IsNullOrEmpty(folderPath))
            {
                GUILayout.Label($"Selected Folder: {folderPath}", EditorStyles.wordWrappedLabel);
            }

            window_length = EditorGUILayout.IntField("Window Length (frames)", window_length);
            window_overlap = EditorGUILayout.IntField("Window Overlap (frames)", window_overlap);

            GUILayout.Label("CSV Settings", EditorStyles.boldLabel);
            filename = EditorGUILayout.TextField("Filename", filename);

            if (GUILayout.Button("Export Data"))
            {
                if (mocapDataContainers == null || mocapDataContainers.Length == 0)
                {
                    Debug.LogError("No Mocap Data Containers found in the selected folder.");
                    return;
                }
                ExportDataAsync();
            }
        }

        private void LoadMocapDataHoldersFromFolder()
        {
            if (string.IsNullOrEmpty(folderPath)) return;

            string[] guids = AssetDatabase.FindAssets("t:MocapData_LiteHolder", new[] { folderPath });
            mocapDataContainers = new MocapData_LiteHolder[guids.Length];

            for (int i = 0; i < guids.Length; i++)
            {
                string assetPath = AssetDatabase.GUIDToAssetPath(guids[i]);
                mocapDataContainers[i] = AssetDatabase.LoadAssetAtPath<MocapData_LiteHolder>(assetPath);
            }

            Debug.Log($"{mocapDataContainers.Length} MocapData_LiteHolder(s) found in folder: {folderPath}");
        }

        private async void ExportDataAsync()
        {
            string baseFileName = $"{filename}_{DateTime.Today:yyyy-MM-dd}_W{window_length}_O{window_overlap}";
            string path = Path.Combine(Application.streamingAssetsPath, baseFileName + ".csv");
            string labelPath = Path.Combine(Application.streamingAssetsPath, baseFileName + "_labels.csv");
            string lookupPath = Path.Combine(Application.streamingAssetsPath, baseFileName + "_lookup_table.csv");

            Stopwatch masterStopwatch = new Stopwatch();
            masterStopwatch.Start();

            int windowId = 0;
            int frameCounter = 0;
            var dataString = new StringBuilder();
            var labelString = new StringBuilder();
            var lookupString = new StringBuilder("Window_ID,AnimationClipList_ID,Clip_ID,Start_Frame,End_Frame\n");

            foreach (var mocapDataHolder in mocapDataContainers)
            {
                var result = await BuildFileContent(mocapDataHolder, windowId, frameCounter);
                dataString.Append(result.data);
                labelString.Append(result.labels);
                lookupString.Append(result.lookupEntries);

                windowId = result.finalWindowId;
                frameCounter = result.finalFrameCounter;
            }

            await WriteToFileAsync(path, dataString.ToString());
            await WriteToFileAsync(labelPath, labelString.ToString());
            await WriteToFileAsync(lookupPath, lookupString.ToString());

            masterStopwatch.Stop();
            Debug.Log($">>> DURATION OF RUN: {masterStopwatch.Elapsed.TotalSeconds} seconds");

            AssetDatabase.Refresh();
        }

        private async Task<(string data, string labels, string lookupEntries, int finalWindowId, int finalFrameCounter)> BuildFileContent(
            MocapData_LiteHolder mocapDataHolder, int startingWindowId, int startingFrameCounter)
        {
            MocapData_Lite mocap = mocapDataHolder.mocapData;
            AnimationClipList clipList = FindClipListForMocapData(mocapDataHolder);

            if (clipList == null)
            {
                Debug.LogError($"Could not find AnimationClipList for {mocapDataHolder.name}");
                return (string.Empty, string.Empty, string.Empty, startingWindowId, startingFrameCounter);
            }

            var sw = new Stopwatch();
            sw.Start();

            var headers = mocap.data_header;
            var data = mocap.data;
            var sequenceLengths = mocap.framesPerSequence;

            var dataString = new StringBuilder();
            var labelString = new StringBuilder();
            var lookupString = new StringBuilder();

            int windowId = startingWindowId;
            int frameCounter = startingFrameCounter;

            if (windowId == 0)
            {
                var headerString = new StringBuilder("Sequence_ID,");

                for (var i = 0; i < headers.Length; i++)
                {
                    if (i == 0)
                    {
                        headerString.Append($"{headers[i]}_Xpos,{headers[i]}_Ypos,{headers[i]}_Zpos,{headers[i]}_Xrot,{headers[i]}_Yrot,{headers[i]}_Zrot,");
                    }
                    else
                    {
                        headerString.Append($"{headers[i]}_Xdir,{headers[i]}_Ydir,{headers[i]}_Zdir,");
                    }
                }

                dataString.AppendLine(headerString.ToString());
                labelString.AppendLine("Sequence_ID,Labels");
            }

            int windowStartIndex = 0;
            int step = window_length - window_overlap;

            await Task.Run(async () =>
            {
                for (var j = 0; j < mocap.sequenceCount; j++)
                {
                    var seqLength = sequenceLengths[j];
                    var clipId = j; // Assuming each sequence represents a clip
                    var windowIndex = 0;

                    while (windowIndex < seqLength - window_length)
                    {
                        int startFrame = windowStartIndex + windowIndex;
                        int endFrame = startFrame + window_length - 1;

                        for (int i = windowIndex; i < windowIndex + window_length; i++)
                        {
                            var index = windowStartIndex + i;
                            var d = data[index];
                            dataString.AppendLine($"{windowId},{d}");
                            labelString.AppendLine($"{windowId},{clipList.classes[d.label_id]}");
                            frameCounter++;

                            await Task.Yield();
                        }

                        lookupString.AppendLine($"{windowId},{mocapDataHolder.name},{clipId},{startFrame},{endFrame}");
                        windowIndex += step;
                        windowId++;
                    }

                    windowStartIndex += seqLength;
                }
            });

            sw.Stop();
            return (dataString.ToString(), labelString.ToString(), lookupString.ToString(), windowId, frameCounter);
        }

        private AnimationClipList FindClipListForMocapData(MocapData_LiteHolder mocapDataHolder)
        {
            string mocapName = mocapDataHolder.name;
            string[] guids = AssetDatabase.FindAssets($"t:AnimationClipList {mocapName}");

            foreach (string guid in guids)
            {
                string path = AssetDatabase.GUIDToAssetPath(guid);
                AnimationClipList clipList = AssetDatabase.LoadAssetAtPath<AnimationClipList>(path);
                if (clipList != null && clipList.name == mocapName)
                {
                    return clipList;
                }
            }

            Debug.LogError($"Could not find AnimationClipList named {mocapName} for MocapData_LiteHolder {mocapName}");
            return null;
        }

        private async Task WriteToFileAsync(string filePath, string content)
        {
            Debug.Log($">>> Saving frames to: {filePath}");
            await using var streamWriter = new StreamWriter(filePath, false);
            await streamWriter.WriteAsync(content);
        }

        private string GetRelativeUnityPath(string fullPath)
        {
            if (fullPath.StartsWith(Application.dataPath))
            {
                return "Assets" + fullPath.Substring(Application.dataPath.Length);
            }

            return fullPath;
        }
    }
    #endif
}

Explanation of Changes

1. Return Tuple Instead of ref Parameters:

   • The BuildFileContent method now returns a tuple containing the updated data, labels, lookup entries, and final values for windowId and frameCounter.

2. Update windowId and frameCounter After Each Task Completion:

   • After each BuildFileContent call, windowId and frameCounter are updated based on the returned values to ensure continuity across tasks.

You

I now need to parse this csv and create a Dictionary in runtime

ChatGPT

To parse the lookup table CSV and create a Dictionary at runtime, you can use C#‘s built-in file handling and string manipulation functions. Assuming the CSV has columns: Window_ID,AnimationClipList_ID,Clip_ID,Start_Frame,End_Frame, we’ll create a Dictionary<int, LookupEntry> where int is the Window_ID and LookupEntry is a struct or class to hold the related information.

Here’s how to do it:

1. Define a LookupEntry Struct: This will hold the AnimationClipList_ID, Clip_ID, Start_Frame, and End_Frame.
2. Read the CSV File: Use File.ReadAllLines or StreamReader to read the CSV file at runtime.
3. Parse Each Line: Split each line by commas, then create LookupEntry instances and populate the dictionary.

Step-by-Step Code

using System;
using System.Collections.Generic;
using System.IO;
using UnityEngine;

public class CsvLookupLoader : MonoBehaviour
{
    private Dictionary<int, LookupEntry> lookupTable;

    [Serializable]
    public struct LookupEntry
    {
        public string AnimationClipListID;
        public int ClipID;
        public int StartFrame;
        public int EndFrame;

        public LookupEntry(string animationClipListID, int clipID, int startFrame, int endFrame)
        {
            AnimationClipListID = animationClipListID;
            ClipID = clipID;
            StartFrame = startFrame;
            EndFrame = endFrame;
        }
    }

    public void LoadLookupTable(string filePath)
    {
        lookupTable = new Dictionary<int, LookupEntry>();

        try
        {
            // Read all lines from the CSV file
            string[] lines = File.ReadAllLines(filePath);

            // Skip the header (assuming the first line is a header)
            for (int i = 1; i < lines.Length; i++)
            {
                string[] values = lines[i].Split(',');

                // Parse values assuming the order: Window_ID,AnimationClipList_ID,Clip_ID,Start_Frame,End_Frame
                int windowId = int.Parse(values[0]);
                string animationClipListID = values[1];
                int clipID = int.Parse(values[2]);
                int startFrame = int.Parse(values[3]);
                int endFrame = int.Parse(values[4]);

                // Create a new LookupEntry
                LookupEntry entry = new LookupEntry(animationClipListID, clipID, startFrame, endFrame);

                // Add to the dictionary
                lookupTable[windowId] = entry;
            }

            Debug.Log($"Successfully loaded lookup table with {lookupTable.Count} entries.");
        }
        catch (Exception ex)
        {
            Debug.LogError($"Failed to load lookup table: {ex.Message}");
        }
    }

    public LookupEntry GetLookupEntry(int windowId)
    {
        if (lookupTable.TryGetValue(windowId, out LookupEntry entry))
        {
            return entry;
        }
        else
        {
            Debug.LogError($"Lookup entry not found for Window ID: {windowId}");
            return default;
        }
    }
}

Explanation of the Code

1. LookupEntry Struct:

   • This struct holds AnimationClipListID, ClipID, StartFrame, and EndFrame as defined in the lookup CSV file.

2. LoadLookupTable Method:

   • This method reads all lines from the specified CSV file, skips the header, and parses each row.
   • Each row is split by commas, and values are converted to the appropriate data type (e.g., int for Window_ID).
   • A new LookupEntry instance is created for each row, and it’s added to lookupTable with Window_ID as the key.

3. GetLookupEntry Method:

   • This method retrieves an entry by Window_ID. It uses TryGetValue to safely check if the Window_ID exists in the dictionary before returning the entry.

Usage Example

1. Load the Table at Runtime:

   • Place this script on a GameObject in your Unity scene, and call LoadLookupTable(filePath) with the path to your lookup CSV.

2. Access Entries:

   • Use GetLookupEntry(windowId) to retrieve specific entries by Window_ID.

Notes

• File Path: filePath should point to the CSV in the StreamingAssets folder, accessible at runtime.
• Error Handling: Basic error handling is added to catch issues during file reading and parsing.

You

is there a faster way?

ChatGPT

Yes, we can improve performance by using a StreamReader with Dictionary initialization. StreamReader offers more control over file reading, which can be faster for larger files than File.ReadAllLines. Also, since this is a CSV file with known data types, we can parse it directly without relying on intermediate data structures like arrays of strings.

Here’s a faster way to load the CSV into a Dictionary:

1. Use StreamReader to process the file line-by-line.
2. Parse and store entries directly into the dictionary without creating extra collections.

Optimized Code Using StreamReader

using System;
using System.Collections.Generic;
using System.IO;
using UnityEngine;

public class CsvLookupLoader : MonoBehaviour
{
    private Dictionary<int, LookupEntry> lookupTable;

    [Serializable]
    public struct LookupEntry
    {
        public string AnimationClipListID;
        public int ClipID;
        public int StartFrame;
        public int EndFrame;

        public LookupEntry(string animationClipListID, int clipID, int startFrame, int endFrame)
        {
            AnimationClipListID = animationClipListID;
            ClipID = clipID;
            StartFrame = startFrame;
            EndFrame = endFrame;
        }
    }

    public void LoadLookupTable(string filePath)
    {
        lookupTable = new Dictionary<int, LookupEntry>();

        try
        {
            using (var reader = new StreamReader(filePath))
            {
                // Skip the header
                reader.ReadLine();

                while (!reader.EndOfStream)
                {
                    string line = reader.ReadLine();

                    // Split the line by comma and parse fields directly
                    int comma1 = line.IndexOf(',');
                    int comma2 = line.IndexOf(',', comma1 + 1);
                    int comma3 = line.IndexOf(',', comma2 + 1);
                    int comma4 = line.IndexOf(',', comma3 + 1);

                    int windowId = int.Parse(line.Substring(0, comma1));
                    string animationClipListID = line.Substring(comma1 + 1, comma2 - comma1 - 1);
                    int clipID = int.Parse(line.Substring(comma2 + 1, comma3 - comma2 - 1));
                    int startFrame = int.Parse(line.Substring(comma3 + 1, comma4 - comma3 - 1));
                    int endFrame = int.Parse(line.Substring(comma4 + 1));

                    // Create and add the entry directly to the dictionary
                    lookupTable[windowId] = new LookupEntry(animationClipListID, clipID, startFrame, endFrame);
                }
            }

            Debug.Log($"Successfully loaded lookup table with {lookupTable.Count} entries.");
        }
        catch (Exception ex)
        {
            Debug.LogError($"Failed to load lookup table: {ex.Message}");
        }
    }

    public LookupEntry GetLookupEntry(int windowId)
    {
        if (lookupTable.TryGetValue(windowId, out LookupEntry entry))
        {
            return entry;
        }
        else
        {
            Debug.LogError($"Lookup entry not found for Window ID: {windowId}");
            return default;
        }
    }
}

Explanation of the Optimizations

1. StreamReader:

   • Using StreamReader to read the file line-by-line is memory-efficient and reduces overhead for large files, as opposed to loading the entire file into memory.

2. Manual String Parsing:

   • Instead of using Split, we locate commas manually and extract substrings for each field. This avoids creating temporary arrays and reduces memory allocation.

3. Direct Dictionary Insertion:

   • Each entry is created and added to lookupTable immediately, reducing temporary data handling.

4. Reduced Exception Handling:

   • Since this is a structured file, we assume well-formed CSV input. More complex handling can be added if necessary, but keeping it minimal enhances performance.

Why This is Faster

• Reduced Memory Allocation: By avoiding intermediate collections (like arrays from Split), this approach allocates less memory and reduces garbage collection.
• Direct Parsing and Insertion: Manually locating commas and parsing values directly avoids unnecessary data transformations.

This optimized method should significantly speed up loading, especially with large CSV files, while using less memory.