IDSolutions/ramdb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 19 Wiki Activity

feat: Improve chunking, fix handler, and add save backup
All checks were successful
Build / Build (push) Successful in 29s
Details

Browse Source 
This commit includes several improvements and fixes:

*   **Improved Chunking Logic:** Refactored the chunking logic in `Utils.cs` to use a more efficient approach, improving performance.  The previous implementation was complex and less performant.  This change simplifies the logic and improves efficiency.
*   **Fixed Handler Function:**  Modified `fnc_handler.sqf` to correctly handle remote execution with NetIds, ensuring that if the NetId resolves to a null object, the function falls back to local execution. This prevents errors when the target object is no longer valid.
*   **Added Save Backup Option:**  Added the ability to create a backup when saving data in `fnc_save.sqf`.  This allows for data recovery in case of corruption.
*   **Updated DLL/SO:** Updated the ArmaRAMDb_x64.dll and ArmaRAMDb_x64.so files.
This commit is contained in:
Jacob Schmidt 2025-03-28 09:45:34 -05:00
parent 2b9ac44516
commit 06bbe231e4
8 changed files with 57 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
ArmaRAMDb_x64.dll
View File

Binary file not shown.

BIN
ArmaRAMDb_x64.so
View File

Binary file not shown.

27
addons/db/functions/fnc_fetch.sqf
View File

@ -26,6 +26,7 @@
params ["_uniqueID", "_function", "_index", "_total", "_datachunk", "_call", "_netId"];
private _dataString = "";
private _index_array = [];
private _count_total = -1;
@ -43,35 +44,15 @@ _count_total = {
if (_count_total == _total) then {
_index_array sort true;
private _allElements = [];
{
private _chunkData = _x select 1;
private _chunkArray = [];
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: Processing chunk: %1", _chunkData];
#endif
try {
_chunkArray = parseSimpleArray _chunkData;
_allElements append _chunkArray;
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: Parsed chunk successfully with %1 elements", count _chunkArray];
#endif
} catch {
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: Error parsing chunk: %1", _chunkData];
#endif
};
_dataString = _dataString + (_x select 1);
} forEach _index_array;
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: 'ramdb_db_fnc_fetch' Combined %1 chunks into array with %2 elements", count _index_array, count _allElements];
diag_log text format ["ArmaRAMDb: 'ramdb_db_fnc_fetch' Data String: %1", _dataString];
#endif
[_uniqueID, _function, _call, _allElements, _netId] call FUNC(handler);
[_uniqueID, _function, _call, (parseSimpleArray _dataString), _netId] call FUNC(handler);
ramdb_db_fetch_array = ramdb_db_fetch_array select {!((_x select 0) in [_uniqueID])};
};

30
addons/db/functions/fnc_handler.sqf
View File

@ -44,10 +44,34 @@ if (_function == "" || count _data == 0) exitWith {
private _func = call compile format ["%1", _function];
if ((!isNil "_netId") and (_netId isNotEqualTo "")) then {
if (_netId != "") then {
private _target = objectFromNetId _netId;
if (_call) then { _data remoteExecCall [_function, _target, false]; } else { _data remoteExec [_function, _target, false]; };
if (!isNull _target) then {
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: 'ramdb_db_fnc_handler' Using NetId: '%1'", _netId];
#endif
if (_call) then {
_data remoteExecCall [_function, _target, false];
} else {
_data remoteExec [_function, _target, false];
};
} else {
#ifdef __A3__DEBUG__
diag_log text format ["ArmaRAMDb: 'ramdb_db_fnc_handler' NetId '%1' resolved to null object, using local execution", _netId];
#endif
if (_call) then {
_data call _func;
} else {
_data spawn _func;
};
};
} else {
if (_call) then { _data call _func; } else { _data spawn _func; };
if (_call) then {
_data call _func;
} else {
_data spawn _func;
};
};

2
addons/db/functions/fnc_save.sqf
View File

@ -30,4 +30,4 @@
params [["_createBackup", false, [false]]];
"ArmaRAMDb" callExtension ["save", []];
"ArmaRAMDb" callExtension ["save", [_createBackup]];

BIN
extension/bin/Release/net8.0/linux-x64/publish/ArmaRAMDb_x64.so
View File

Binary file not shown.

BIN
extension/bin/Release/net8.0/win-x64/publish/ArmaRAMDb_x64.dll
View File

Binary file not shown.

182
extension/src/Utils.cs
View File

@ -24,111 +24,53 @@ namespace ArmaRAMDb
return str;
}
public static List<string> SplitIntoChunks(string data, int maxChunkSize)
public static List<string> SplitIntoChunks(string data, int chunkSize)
{
int chunksNeeded = (int)Math.Ceiling((double)data.Length / chunkSize);
List<string> chunks = [];
if (data.StartsWith("[") && data.EndsWith("]"))
for (int i = 0; i < chunksNeeded; i++)
{
string innerData = data[1..^1];
List<string> elements = [];
int depth = 0;
int startPos = 0;
for (int i = 0; i < innerData.Length; i++)
{
char c = innerData[i];
if (c == '[') depth++;
else if (c == ']') depth--;
else if (c == ',' && depth == 0)
{
elements.Add(innerData[startPos..i]);
startPos = i + 1;
}
}
if (startPos < innerData.Length)
{
elements.Add(innerData[startPos..]);
}
StringBuilder currentChunk = new StringBuilder();
foreach (string element in elements)
{
if (currentChunk.Length > 0 &&
Encoding.UTF8.GetByteCount(currentChunk.ToString() + "," + element) > maxChunkSize)
{
chunks.Add(currentChunk.ToString());
currentChunk.Clear();
}
if (currentChunk.Length > 0)
{
currentChunk.Append(",");
}
currentChunk.Append(element);
}
if (currentChunk.Length > 0)
{
chunks.Add(currentChunk.ToString());
}
}
else
{
int bytesProcessed = 0;
while (bytesProcessed < Encoding.UTF8.GetByteCount(data))
{
int charCount = 0;
while (bytesProcessed + Encoding.UTF8.GetByteCount(data.Substring(bytesProcessed, Math.Min(charCount + 1, data.Length - bytesProcessed))) <= maxChunkSize &&
bytesProcessed + charCount < data.Length)
{
charCount++;
}
chunks.Add(data.Substring(bytesProcessed, charCount));
bytesProcessed += charCount;
}
int start = i * chunkSize;
int end = Math.Min(data.Length, start + chunkSize);
chunks.Add(data[start..end]);
}
return chunks;
}
public static long GetUniqueId()
{
return DateTimeOffset.Now.ToUnixTimeMilliseconds();
}
public static void CheckByteCount(string uniqueId, string data, string function, string entity, bool call, int bufferSize)
{
if (Encoding.UTF8.GetByteCount(data) <= bufferSize)
{
if (!data.StartsWith('['))
data = BuildArray(data);
Main.Log($"Single chunk data: {data}", "debug");
string dataAsString = $"[\"{uniqueId}\",\"{function}\",{call.ToString().ToLower()},{data},\"{entity}\"]";
Main.Log($"Single chunk data string: {dataAsString}", "debug");
Main.Callback("ArmaRAMDb", "ramdb_db_fnc_handler", dataAsString);
}
else
{
if (!data.StartsWith('[') || !data.EndsWith(']'))
{
data = BuildArray(data);
}
List<string> elements = ParseArrayElements(data);
Main.Log($"Single chunk data: {data}", "debug");
int totalChunks = CalculateChunks(elements, bufferSize);
var chunks = SplitIntoChunks(data, bufferSize);
int totalChunks = chunks.Count;
for (int chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++)
for (int chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++)
{
List<string> chunkElements = GetChunkElements(elements, chunkIndex, totalChunks);
string chunkData = "[" + string.Join(",", chunkElements) + "]";
string escapedChunkData = chunkData.Replace("\"", "\"\"");
string escapedChunkData = chunks[chunkIndex].Replace("\"", "\"\"");
string chunkAsString = $"[\"{uniqueId}\",\"{function}\",{chunkIndex+1},{totalChunks},\"{escapedChunkData}\",{call.ToString().ToLower()},\"{entity}\"]";
Main.Log($"Sending chunk {chunkIndex+1}/{totalChunks}: {chunkAsString}", "debug");
@ -136,91 +78,5 @@ namespace ArmaRAMDb
}
}
}
private static List<string> ParseArrayElements(string arrayString)
{
List<string> elements = [];
string content = arrayString[1..^1].Trim();
if (string.IsNullOrEmpty(content))
return elements;
int startPos = 0;
int nestLevel = 0;
bool inQuotes = false;
for (int i = 0; i < content.Length; i++)
{
char c = content[i];
if (c == '\\' && i + 1 < content.Length && content[i + 1] == '"')
{
i++;
continue;
}
if (c == '"')
inQuotes = !inQuotes;
if (!inQuotes)
{
if (c == '[') nestLevel++;
else if (c == ']') nestLevel--;
if (nestLevel == 0 && c == ',')
{
elements.Add(content[startPos..i].Trim());
startPos = i + 1;
}
}
}
if (startPos < content.Length)
{
elements.Add(content[startPos..].Trim());
}
return elements;
}
private static int CalculateChunks(List<string> elements, int bufferSize)
{
int chunks = 1;
int currentSize = 2;
foreach (string element in elements)
{
int elementSize = Encoding.UTF8.GetByteCount(element) + 1;
if (currentSize + elementSize > bufferSize)
{
chunks++;
currentSize = 2 + elementSize;
}
else
{
currentSize += elementSize;
}
}
return chunks;
}
private static List<string> GetChunkElements(List<string> allElements, int chunkIndex, int totalChunks)
{
int elementsPerChunk = (int)Math.Ceiling((double)allElements.Count / totalChunks);
int startIdx = chunkIndex * elementsPerChunk;
int count = Math.Min(elementsPerChunk, allElements.Count - startIdx);
List<string> result = [];
for (int i = 0; i < count; i++)
{
if (startIdx + i < allElements.Count)
result.Add(allElements[startIdx + i]);
}
return result;
}
}
}