Compare commits

...

3 Commits

Author SHA1 Message Date
github-actions
fe5268f374 v1.0.0 Build 19 2025-05-04 00:34:29 +00:00
Jacob Schmidt
1d5225f279 feat: Implement binary serialization for RAM database and mission end save
All checks were successful
Build / Build (push) Successful in 29s
This commit introduces binary serialization for the RAM database to improve performance and adds a mission end event handler to ensure data is saved with a backup when the mission ends.

- Implemented binary serialization and deserialization for key-values, hash tables, and lists in `Utils.cs`. This replaces the previous text-based serialization, resulting in faster save and load times.
- Moved data writing and reading logic from `RAMDb.cs` to `Utils.cs` for better code organization and reusability.
- Modified `RAMDb.cs` to use the new binary serialization methods (`ExportToBinary` and `ImportFromBinary`).
- Added a mission end event handler in `XEH_preInit_server.sqf` to trigger a forced save with backup when the mission ends. This ensures that data is not lost in case of unexpected server shutdowns.
- Updated the extension binaries (`ArmaRAMDb_x64.dll` and `ArmaRAMDb_x64.so`) with the new serialization logic.
- Added a static `db` property to the `Main` class to allow access to the `RAMDb` instance from anywhere in the extension.
- Modified the `Save` and `Load` methods in `Main.cs` to use the new binary serialization methods.
- Updated the `Dispose` method in `RAMDb.cs` to ensure a final save and backup is performed when the object is disposed.
2025-05-03 19:34:15 -05:00
Jacob Schmidt
5ed8f74c71 Removed sqf_validator.py 2025-04-05 14:32:49 -05:00
10 changed files with 147 additions and 317 deletions

Binary file not shown.

Binary file not shown.

View File

@ -1 +1,6 @@
#include "script_component.hpp"
#include "script_component.hpp"
addMissionEventHandler ["Ended", {
"ArmaRAMDb" callExtension ["save", [true]];
INFO("Mission ended - forced save with backup");
}];

View File

@ -1,4 +1,4 @@
#define MAJOR 1
#define MINOR 0
#define PATCH 0
#define BUILD 18
#define BUILD 19

View File

@ -19,6 +19,9 @@ namespace ArmaRAMDb
{
private const string ARDB_VERSION = "1.0.0";
public const int ARDB_BUFFERSIZE = 10240;
#pragma warning disable CA2211 // Non-constant fields should not be visible
public static RAMDb db;
#pragma warning restore CA2211 // Non-constant fields should not be visible
public static readonly string DEFAULT_ARDB_PATH = $"@ramdb{Path.DirectorySeparatorChar}ArmaRAMDb.ardb";
public static string ARDB_LOGFOLDER { get; private set; } = $"{Path.DirectorySeparatorChar}@ramdb{Path.DirectorySeparatorChar}logs";
public static bool ARDB_DEBUG {get; private set; } = false;
@ -81,12 +84,13 @@ namespace ArmaRAMDb
Log($"Config file found! Context Mode: {ARDB_CONTEXTLOG}! Debug Mode: {ARDB_DEBUG}! " +
$"Auto Backup: {RAMDb.AutoBackupEnabled} (every {RAMDb.BackupFrequencyMinutes} min, keep {RAMDb.MaxBackupsToKeep})", "action");
db ??= new RAMDb();
// First, load any existing RDB file
var db = new RAMDb();
if (File.Exists(Path.Combine(Environment.CurrentDirectory, DEFAULT_ARDB_PATH)))
{
db.ImportFromArdb();
db.ImportFromBinary();
ARDB_ISLOADED = true;
Log("Existing ARDB data loaded during initialization", "action");
}
@ -321,7 +325,7 @@ namespace ArmaRAMDb
WriteOutput(output, "Data loaded");
return 100;
case "save":
Save(args);
Save(args, argc);
WriteOutput(output, "Data saved");
return 100;
case "set":
@ -384,23 +388,23 @@ namespace ArmaRAMDb
Marshal.Copy(bytes, 0, (nint)output, bytes.Length);
}
private static void Save(List<string> args)
private static void Save(List<string> args, int argc)
{
var db = new RAMDb();
db ??= new RAMDb();
// Convert string to boolean properly
bool createBackup = args.Count > 0 &&
bool createBackup = argc > 0 &&
(args[0].Trim('"').Equals("true", StringComparison.CurrentCultureIgnoreCase) || args[0].Trim('"') == "1");
db.ExportToArdb(createBackup);
db.ExportToBinary(createBackup);
Log($"Data saved to ARDB{(createBackup ? " with backup" : "")}", "action");
}
private static void Load()
{
var db = new RAMDb();
db ??= new RAMDb();
db.ImportFromArdb();
db.ImportFromBinary();
ARDB_ISLOADED = true;
Log("Data loaded from ARDB", "action");
}
@ -442,7 +446,7 @@ namespace ArmaRAMDb
await HashStore.HashGet(STEAMID, args[0].Trim('"'), args[1].Trim('"'), _uniqueID, args[2].Trim('"'), args[3].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for HashGet operation", "warning");
Log($"Unexpected argument count: {argc} for HashGet operation", "warning");
break;
}
});
@ -465,7 +469,7 @@ namespace ArmaRAMDb
await HashStore.HashGet(args[0].Trim('"'), args[1].Trim('"'), args[2].Trim('"'), _uniqueID, args[3].Trim('"'), args[4].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for HashGetId operation", "warning");
Log($"Unexpected argument count: {argc} for HashGetId operation", "warning");
break;
}
});
@ -488,7 +492,7 @@ namespace ArmaRAMDb
await HashStore.HashGetAllAsync(STEAMID, args[0].Trim('"'), _uniqueID, args[1].Trim('"'), args[2].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for HashGetAll operation", "warning");
Log($"Unexpected argument count: {argc} for HashGetAll operation", "warning");
break;
}
});
@ -511,7 +515,7 @@ namespace ArmaRAMDb
await HashStore.HashGetAllAsync(args[0].Trim('"'), args[1].Trim('"'), _uniqueID, args[2], args[3].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for HashGetAllId operation", "warning");
Log($"Unexpected argument count: {argc} for HashGetAllId operation", "warning");
break;
}
});
@ -555,7 +559,7 @@ namespace ArmaRAMDb
await HashStore.HashSetAsync(STEAMID, args[0].Trim('"'), args[1].Trim('"'));
break;
default:
Main.Log($"Invalid argument count: {argc} for HashSet operation", "warning");
Log($"Invalid argument count: {argc} for HashSet operation", "warning");
break;
}
});
@ -579,7 +583,7 @@ namespace ArmaRAMDb
await HashStore.HashSetAsync(args[0].Trim('"'), args[1].Trim('"'), args[2].Trim('"'));
break;
default:
Main.Log($"Invalid argument count: {argc} for HashSetId operation", "warning");
Log($"Invalid argument count: {argc} for HashSetId operation", "warning");
break;
}
});
@ -601,7 +605,7 @@ namespace ArmaRAMDb
await ListStore.ListAddAsync(args[0].Trim('"'), args[1].Trim('"'));
break;
default:
Main.Log($"Invalid argument count: {argc} for ListAdd operation", "warning");
Log($"Invalid argument count: {argc} for ListAdd operation", "warning");
break;
}
});
@ -643,7 +647,7 @@ namespace ArmaRAMDb
await ListStore.ListIndexAsync(args[0].Trim('"'), args[1].Trim('"'), args[2].Trim('"'), _uniqueID, args[3].Trim('"'), args[4].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for ListIdx operation", "warning");
Log($"Unexpected argument count: {argc} for ListIdx operation", "warning");
break;
}
});
@ -666,7 +670,7 @@ namespace ArmaRAMDb
await ListStore.ListRangeAsync(args[0].Trim('"'), args[1].Trim('"'), args[2].Trim('"'), args[3].Trim('"'), _uniqueID, args[4].Trim('"'), args[5].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for ListRng operation", "warning");
Log($"Unexpected argument count: {argc} for ListRng operation", "warning");
break;
}
});
@ -709,7 +713,7 @@ namespace ArmaRAMDb
await KeyValueStore.GetAsync(args[0].Trim('"'), args[1].Trim('"'), _uniqueID, args[2].Trim('"'), args[3].Trim('"'));
break;
default:
Main.Log($"Unexpected argument count: {argc} for Get operation", "warning");
Log($"Unexpected argument count: {argc} for Get operation", "warning");
break;
}
});
@ -727,7 +731,7 @@ namespace ArmaRAMDb
private static string HandleListBackupsOperation()
{
var db = new RAMDb();
db ??= new RAMDb();
var backups = db.ListBackups();
if (backups.Count > 0)
@ -752,7 +756,7 @@ namespace ArmaRAMDb
if (args.Count < 1) return false;
string backupFileName = args[0].Trim('"');
var db = new RAMDb();
db ??= new RAMDb();
var backups = db.ListBackups();
// Find the full path based on filename
@ -774,7 +778,7 @@ namespace ArmaRAMDb
if (args.Count < 1) return false;
string backupFileName = args[0].Trim('"');
var db = new RAMDb();
db ??= new RAMDb();
var backups = db.ListBackups();
// Find the full path based on filename

View File

@ -4,7 +4,7 @@ using System.Collections.Concurrent;
namespace ArmaRAMDb
#pragma warning restore IDE0130 // Namespace does not match folder structure
{
internal class RAMDb(string ardbPath = null) : IDisposable
public class RAMDb(string ardbPath = null) : IDisposable
{
private readonly string _ardbPath = Path.Combine(Environment.CurrentDirectory, ardbPath ?? Main.DEFAULT_ARDB_PATH);
public static readonly ConcurrentDictionary<string, string> _keyValues = new();
@ -16,7 +16,7 @@ namespace ArmaRAMDb
public static int MaxBackupsToKeep { get; set; } = 10;
private static Timer _backupTimer;
public void ImportFromArdb()
public void ImportFromBinary()
{
try
{
@ -100,7 +100,7 @@ namespace ArmaRAMDb
}
}
public void ExportToArdb(bool createBackup = false)
public void ExportToBinary(bool createBackup = false)
{
try
{
@ -111,7 +111,7 @@ namespace ArmaRAMDb
{
writer.Write(1);
WriteDataToBinaryWriter(writer);
Utils.WriteDataToBinaryWriter(writer);
}
if (createBackup)
@ -126,7 +126,7 @@ namespace ArmaRAMDb
using (var stream = new FileStream(backupPath, FileMode.Create))
using (var writer = new BinaryWriter(stream))
{
WriteDataToBinaryWriter(writer);
Utils.WriteDataToBinaryWriter(writer);
}
Main.Log($"Created backup at: {backupPath}", "debug");
@ -140,44 +140,6 @@ namespace ArmaRAMDb
}
}
private static void WriteDataToBinaryWriter(BinaryWriter writer)
{
// Write KeyValues
writer.Write(_keyValues.Count);
foreach (var pair in _keyValues)
{
writer.Write(pair.Key);
writer.Write(pair.Value);
}
// Write HashTables
writer.Write(_hashTables.Count);
foreach (var table in _hashTables)
{
writer.Write(table.Key);
writer.Write(table.Value.Count);
foreach (var entry in table.Value)
{
writer.Write(entry.Key);
writer.Write(entry.Value);
}
}
// Write Lists
writer.Write(_lists.Count);
foreach (var list in _lists)
{
writer.Write(list.Key);
writer.Write(list.Value.Count);
foreach (var item in list.Value)
{
writer.Write(item);
}
}
}
public List<string> ListBackups()
{
string backupDirectory = Path.Combine(Path.GetDirectoryName(_ardbPath), "backups");
@ -211,7 +173,7 @@ namespace ArmaRAMDb
_hashTables.Clear();
_lists.Clear();
ReadDataFromBinaryReader(reader);
Utils.ReadDataFromBinaryReader(reader);
Main.Log($"Restored from backup: {backupPath}", "info");
return true;
@ -225,62 +187,6 @@ namespace ArmaRAMDb
return false;
}
private static void ReadDataFromBinaryReader(BinaryReader reader)
{
// Read KeyValues
int keyValueCount = reader.ReadInt32();
for (int i = 0; i < keyValueCount; i++)
{
string key = reader.ReadString();
string value = reader.ReadString();
_keyValues.TryAdd(key, value);
Main.Log($"Loaded key-value: {key} = {value[..Math.Min(50, value.Length)]}...", "debug");
}
// Read HashTables
int tableCount = reader.ReadInt32();
for (int i = 0; i < tableCount; i++)
{
string tableName = reader.ReadString();
Main.Log($"Loading table: {tableName}", "debug");
var concurrentDict = new ConcurrentDictionary<string, string>();
int entryCount = reader.ReadInt32();
for (int j = 0; j < entryCount; j++)
{
string key = reader.ReadString();
string value = reader.ReadString();
concurrentDict.TryAdd(key, value);
Main.Log($"Loaded entry: {key} = {value[..Math.Min(50, value.Length)]}...", "debug");
}
_hashTables.TryAdd(tableName, concurrentDict);
}
// Read Lists
int listCount = reader.ReadInt32();
for (int i = 0; i < listCount; i++)
{
string listName = reader.ReadString();
Main.Log($"Loading list: {listName}", "debug");
var items = new List<string>();
int itemCount = reader.ReadInt32();
for (int j = 0; j < itemCount; j++)
{
string value = reader.ReadString();
items.Add(value);
Main.Log($"Loaded item: {value[..Math.Min(50, value.Length)]}...", "debug");
}
_lists.TryAdd(listName, items);
}
Main.Log("ARDB import complete", "debug");
}
public static void InitializeAutoBackup()
{
if (AutoBackupEnabled)
@ -299,8 +205,7 @@ namespace ArmaRAMDb
{
try
{
var db = new RAMDb();
db.ExportToArdb(true);
Main.db.ExportToBinary(true);
ManageBackupRotation();
@ -316,8 +221,7 @@ namespace ArmaRAMDb
{
try
{
var db = new RAMDb();
var backups = db.ListBackups();
var backups = Main.db.ListBackups();
if (backups.Count > MaxBackupsToKeep)
{
@ -336,8 +240,17 @@ namespace ArmaRAMDb
public void Dispose()
{
_backupTimer?.Dispose();
ExportToArdb(createBackup: true);
try
{
ExportToBinary(true);
_backupTimer?.Dispose();
Main.Log("RAMDb disposed with final save and backup", "action");
}
catch (Exception ex)
{
Main.Log($"Error during final save on disposal: {ex.Message}", "error");
}
GC.SuppressFinalize(this);
}
}
}

View File

@ -1,4 +1,5 @@
using System.Text;
using System.Collections.Concurrent;
#pragma warning disable IDE0130 // Namespace does not match folder structure
namespace ArmaRAMDb
@ -44,6 +45,100 @@ namespace ArmaRAMDb
return DateTimeOffset.Now.ToUnixTimeMilliseconds();
}
public static void WriteDataToBinaryWriter(BinaryWriter writer)
{
// Write KeyValues
writer.Write(RAMDb._keyValues.Count);
foreach (var pair in RAMDb._keyValues)
{
writer.Write(pair.Key);
writer.Write(pair.Value);
}
// Write HashTables
writer.Write(RAMDb._hashTables.Count);
foreach (var table in RAMDb._hashTables)
{
writer.Write(table.Key);
writer.Write(table.Value.Count);
foreach (var entry in table.Value)
{
writer.Write(entry.Key);
writer.Write(entry.Value);
}
}
// Write Lists
writer.Write(RAMDb._lists.Count);
foreach (var list in RAMDb._lists)
{
writer.Write(list.Key);
writer.Write(list.Value.Count);
foreach (var item in list.Value)
{
writer.Write(item);
}
}
}
public static void ReadDataFromBinaryReader(BinaryReader reader)
{
// Read KeyValues
int keyValueCount = reader.ReadInt32();
for (int i = 0; i < keyValueCount; i++)
{
string key = reader.ReadString();
string value = reader.ReadString();
RAMDb._keyValues.TryAdd(key, value);
Main.Log($"Loaded key-value: {key} = {value[..Math.Min(50, value.Length)]}...", "debug");
}
// Read HashTables
int tableCount = reader.ReadInt32();
for (int i = 0; i < tableCount; i++)
{
string tableName = reader.ReadString();
Main.Log($"Loading table: {tableName}", "debug");
var concurrentDict = new ConcurrentDictionary<string, string>();
int entryCount = reader.ReadInt32();
for (int j = 0; j < entryCount; j++)
{
string key = reader.ReadString();
string value = reader.ReadString();
concurrentDict.TryAdd(key, value);
Main.Log($"Loaded entry: {key} = {value[..Math.Min(50, value.Length)]}...", "debug");
}
RAMDb._hashTables.TryAdd(tableName, concurrentDict);
}
// Read Lists
int listCount = reader.ReadInt32();
for (int i = 0; i < listCount; i++)
{
string listName = reader.ReadString();
Main.Log($"Loading list: {listName}", "debug");
var items = new List<string>();
int itemCount = reader.ReadInt32();
for (int j = 0; j < itemCount; j++)
{
string value = reader.ReadString();
items.Add(value);
Main.Log($"Loaded item: {value[..Math.Min(50, value.Length)]}...", "debug");
}
RAMDb._lists.TryAdd(listName, items);
}
Main.Log("ARDB import complete", "debug");
}
public static void CheckByteCount(string uniqueId, string data, string function, string entity, bool call, int bufferSize)
{
if (Encoding.UTF8.GetByteCount(data) <= bufferSize)

View File

@ -1,187 +0,0 @@
#!/usr/bin/env python3
import fnmatch
import os
import re
import sys
def valid_keyword_after_code(content, index):
for word in ["for", "do", "count", "each", "forEach", "else", "and", "not", "isEqualTo", "isNotEqualTo", "in", "call", "spawn", "execVM", "catch", "param", "select", "apply", "findIf", "remoteExec"]:
if content.find(word, index, index + len(word)) != -1:
return True
return False
def check_sqf(filepath):
errors = []
with open(filepath, "r", encoding = "utf-8", errors = "ignore") as file:
content = file.read()
# Store all brackets we find in this file, so we can validate everything on the end
brackets = []
# Used in case we are in a line comment (//)
ignore_till_eol = False
# To check if we are in a comment block
in_comment_block = False
check_if_comment = False
# Used in case we are in a comment block (/* */)
# This is true if we detect a * inside a comment block
# If the next character is a /, it means we end our comment block
check_if_closing = False
# We ignore everything inside a string
in_string = False
# Used to store the starting type of a string, so we can match that to the end of a string
string_type = ""
# Used to check for semicolon after code blocks
last_is_curly_brace = False
check_for_semicolon = False
# Extra information so we know what line we find errors at
line_number = 1
char_index = 0
for c in content:
if last_is_curly_brace:
last_is_curly_brace = False
# Test generates false positives with binary commands that take CODE as 2nd arg (e.g. findIf)
check_for_semicolon = not re.search("findIf", content, re.IGNORECASE)
# Keep track of current line number
if c == "\n":
line_number += 1
# While we are in a string, we can ignore everything else, except the end of the string
if in_string:
if c == string_type:
in_string = False
# Look for the end of this comment block
elif in_comment_block:
if c == "*":
check_if_closing = True
elif check_if_closing:
if c == "/":
in_comment_block = False
elif c != "*":
check_if_closing = False
# If we are not in a comment block, we will check if we are at the start of one or count the () {} and []
else:
# This means we have encountered a /, so we are now checking if this is an inline comment or a comment block
if check_if_comment:
check_if_comment = False
# If the next character after / is a *, we are at the start of a comment block
if c == "*":
in_comment_block = True
# Otherwise, check if we are in an line comment, / followed by another / (//)
elif c == "/":
ignore_till_eol = True
if not in_comment_block:
if ignore_till_eol:
# We are in a line comment, just continue going through the characters until we find an end of line
if c == "\n":
ignore_till_eol = False
else:
if c == '"' or c == "'":
in_string = True
string_type = c
elif c == "/":
check_if_comment = True
elif c == "\t":
errors.append(" ERROR: Found a tab on line {}.".format(line_number))
elif c in ["(", "[", "{"]:
brackets.append(c)
elif c == ")":
if not brackets or brackets[-1] in ["[", "{"]:
errors.append(" ERROR: Missing parenthesis '(' on line {}.".format(line_number))
brackets.append(c)
elif c == "]":
if not brackets or brackets[-1] in ["(", "{"]:
errors.append(" ERROR: Missing square bracket '[' on line {}.".format(line_number))
brackets.append(c)
elif c == "}":
last_is_curly_brace = True
if not brackets or brackets[-1] in ["(", "["]:
errors.append(" ERROR: Missing curly brace '{{' on line {}.".format(line_number))
brackets.append(c)
if check_for_semicolon:
# Keep reading until no white space or comments
if c not in [" ", "\t", "\n", "/"]:
check_for_semicolon = False
if c not in ["]", ")", "}", ";", ",", "&", "!", "|", "="] and not valid_keyword_after_code(content, char_index):
errors.append(" ERROR: Possible missing semicolon ';' on line {}.".format(line_number))
char_index += 1
# Compare opening and closing bracket counts
if brackets.count("(") != brackets.count(")"):
errors.append(" ERROR: Unequal number of parentheses, '(' = {}, ')' = {}.".format(brackets.count("("), brackets.count(")")))
if brackets.count("[") != brackets.count("]"):
errors.append(" ERROR: Unequal number of square brackets, '[' = {}, ']' = {}.".format(brackets.count("["), brackets.count("]")))
if brackets.count("{") != brackets.count("}"):
errors.append(" ERROR: Unequal number of curly braces, '{{' = {}, '}}' = {}.".format(brackets.count("{"), brackets.count("}")))
# Ensure includes are before block comments
if re.compile('\s*(/\*[\s\S]+?\*/)\s*#include').match(content):
errors.append(" ERROR: Found an #include after a block comment.")
return errors
def main():
print("Validating SQF")
print("--------------")
# Allow running from root directory and tools directory
root_dir = ".."
if os.path.exists("addons"):
root_dir = "."
# Check all SQF files in the project directory
sqf_files = []
for root, _, files in os.walk(root_dir):
for file in fnmatch.filter(files, "*.sqf"):
sqf_files.append(os.path.join(root, file))
sqf_files.sort()
bad_count = 0
for filepath in sqf_files:
errors = check_sqf(filepath)
if errors:
print("\nFound {} error(s) in {}:".format(len(errors), os.path.relpath(filepath, root_dir)))
for error in errors:
print(error)
bad_count += 1
print("\nChecked {} files, found errors in {}.".format(len(sqf_files), bad_count))
if bad_count == 0:
print("SQF Validation PASSED")
else:
print("SQF Validation FAILED")
return bad_count
if __name__ == "__main__":
sys.exit(main())