commit de14285314211870415aeef455b024b05cca0b8b Author: Jacob Schmidt Date: Thu Apr 10 21:50:41 2025 -0500 Initial Repo Setup diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..9c62458 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,48 @@ +name: Build + +on: + push: + branches: [ master ] + tags: + - 'v*' + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '9.0.x' + + - name: Restore dependencies + run: dotnet restore Firefly.sln + + - name: Build + run: | + chmod +x build-all.sh + ./build-all.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: firefly-artifacts + path: | + artifacts/exe/* + artifacts/native/* + + - name: Create Release + if: startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@v1 + with: + files: | + artifacts/exe/* + artifacts/native/* + generate_release_notes: true \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff7969f --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.editorconfig +nuget.config +*.7z +backups +bin +obj +.venv +.vs +.vscode \ No newline at end of file diff --git a/Firefly.csproj b/Firefly.csproj new file mode 100644 index 0000000..f415398 --- /dev/null +++ b/Firefly.csproj @@ -0,0 +1,62 @@ + + + + + net9.0 + enable + enable + Speed + icon.ico + icon.png + true + win-x64;linux-x64;osx-x64 + true + true + embedded + + + Firefly + Firefly + Firefly + A Redis-compatible server + Firefly + 1.0.0 + IDSolutions + true + false + true + + + + + Exe + true + true + true + true + true + Speed + true + true + + + + + Library + true + Shared + true + true + true + Speed + true + + + + + + + + + + \ No newline at end of file diff --git a/Firefly.csproj.user b/Firefly.csproj.user new file mode 100644 index 0000000..79e89c7 --- /dev/null +++ b/Firefly.csproj.user @@ -0,0 +1,11 @@ + + + + <_LastSelectedProfileId>G:\forge\firefly\extension\Firefly\Properties\PublishProfiles\FolderProfile.pubxml + + + + Designer + + + \ No newline at end of file diff --git a/Firefly.sln b/Firefly.sln new file mode 100644 index 0000000..451a157 --- /dev/null +++ b/Firefly.sln @@ -0,0 +1,30 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Firefly", "Firefly.csproj", "{3DB35960-520A-BD42-07BD-4CAE9A4E697D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Debug|x64.ActiveCfg = Debug|x64 + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Debug|x64.Build.0 = Debug|x64 + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Release|Any CPU.Build.0 = Release|Any CPU + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Release|x64.ActiveCfg = Release|x64 + {3DB35960-520A-BD42-07BD-4CAE9A4E697D}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FD0A49B3-12BC-4356-BD54-4D73658BE95F} + EndGlobalSection +EndGlobal diff --git a/Properties/PublishProfiles/FolderProfile.pubxml b/Properties/PublishProfiles/FolderProfile.pubxml new file mode 100644 index 0000000..d202779 --- /dev/null +++ b/Properties/PublishProfiles/FolderProfile.pubxml @@ -0,0 +1,16 @@ + + + + + Release + Any CPU + bin\Release\net9.0\publish\ + FileSystem + <_TargetId>Folder + net9.0 + win-x64 + true + true + true + + \ No newline at end of file diff --git a/Properties/PublishProfiles/FolderProfile.pubxml.user b/Properties/PublishProfiles/FolderProfile.pubxml.user new file mode 100644 index 0000000..3f159a4 --- /dev/null +++ b/Properties/PublishProfiles/FolderProfile.pubxml.user @@ -0,0 +1,8 @@ + + + + + True|2025-04-08T00:26:53.2010541Z||;True|2025-04-07T19:26:46.3598239-05:00||;True|2025-04-07T19:07:58.7012000-05:00||;True|2025-04-07T19:05:51.2227643-05:00||;True|2025-04-07T19:03:08.0269213-05:00||;True|2025-04-07T18:29:04.5003554-05:00||;True|2025-04-06T23:16:01.4293237-05:00||;False|2025-04-06T23:15:41.6244752-05:00||;True|2025-04-06T21:47:02.7035505-05:00||;True|2025-04-06T20:08:49.6142219-05:00||;True|2025-04-06T19:25:51.2602973-05:00||;True|2025-04-06T19:17:03.2181403-05:00||;True|2025-04-06T19:04:13.8498671-05:00||;True|2025-04-06T19:01:25.0268913-05:00||;True|2025-04-06T18:58:25.6890700-05:00||;True|2025-04-06T18:55:25.6367097-05:00||;True|2025-04-06T18:53:20.2451573-05:00||;True|2025-04-06T18:45:02.8482945-05:00||;False|2025-04-06T18:44:50.0356423-05:00||;True|2025-04-06T18:16:53.9309464-05:00||;True|2025-04-06T17:39:44.9315977-05:00||;False|2025-04-06T17:38:07.1833743-05:00||;True|2025-04-06T17:28:34.7769158-05:00||;True|2025-04-06T17:27:47.9261712-05:00||;True|2025-04-06T17:22:10.3979209-05:00||;True|2025-04-06T17:06:24.3332447-05:00||;True|2025-04-06T16:59:11.3671748-05:00||;True|2025-04-06T16:28:27.6102439-05:00||;True|2025-04-06T16:25:29.8493098-05:00||;True|2025-04-06T16:07:11.4149691-05:00||;True|2025-04-06T15:30:49.5703060-05:00||;True|2025-04-06T14:52:44.3134873-05:00||;True|2025-04-06T14:37:32.1843661-05:00||;True|2025-04-06T13:13:57.7572594-05:00||;True|2025-04-06T13:13:40.5523132-05:00||;False|2025-04-06T13:13:24.6183721-05:00||;True|2025-04-06T12:36:35.2352725-05:00||;True|2025-04-06T00:38:44.2848632-05:00||;False|2025-04-06T00:27:17.9337778-05:00||;False|2025-04-06T00:26:34.3407057-05:00||;False|2025-04-06T00:26:24.6819572-05:00||;True|2025-04-06T00:02:24.4950451-05:00||;True|2025-04-05T23:40:36.1415759-05:00||;True|2025-04-05T22:59:15.6812178-05:00||;True|2025-04-05T22:30:54.7814684-05:00||;True|2025-04-05T22:20:22.3907744-05:00||;True|2025-04-05T22:11:14.6770330-05:00||;True|2025-04-05T22:04:33.3236186-05:00||;True|2025-04-05T22:00:11.5893340-05:00||;True|2025-04-05T21:49:33.4826768-05:00||;True|2025-04-05T21:44:51.9154306-05:00||;True|2025-04-05T21:35:04.6071930-05:00||;True|2025-04-05T21:22:35.7252158-05:00||;True|2025-04-05T21:18:15.7082379-05:00||;True|2025-04-05T20:44:29.2052360-05:00||;True|2025-04-05T20:41:04.4189956-05:00||;True|2025-04-05T20:38:24.1151587-05:00||;True|2025-04-05T20:36:46.0623020-05:00||;True|2025-04-05T20:33:19.4003638-05:00||;True|2025-04-05T20:30:31.7236371-05:00||; + + + \ No newline at end of file diff --git a/Properties/Resources.Designer.cs b/Properties/Resources.Designer.cs new file mode 100644 index 0000000..03dec8e --- /dev/null +++ b/Properties/Resources.Designer.cs @@ -0,0 +1,88 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +using System.CodeDom.Compiler; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Resources; +using System.Runtime.CompilerServices; + +namespace Firefly.Properties { + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [GeneratedCode("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [DebuggerNonUserCode()] + [CompilerGenerated()] + internal class Resources { + + private static ResourceManager resourceMan; + + private static CultureInfo resourceCulture; + + [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [EditorBrowsable(EditorBrowsableState.Advanced)] + internal static ResourceManager ResourceManager { + get { + if (ReferenceEquals(resourceMan, null)) { + ResourceManager temp = new ResourceManager("Firefly.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [EditorBrowsable(EditorBrowsableState.Advanced)] + internal static CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized resource of type System.Byte[]. + /// + internal static byte[] icon { + get { + object obj = ResourceManager.GetObject("icon", resourceCulture); + return ((byte[])(obj)); + } + } + + /// + /// Looks up a localized resource of type System.Byte[]. + /// + internal static byte[] logo { + get { + object obj = ResourceManager.GetObject("logo", resourceCulture); + return ((byte[])(obj)); + } + } + } +} diff --git a/Properties/Resources.resx b/Properties/Resources.resx new file mode 100644 index 0000000..30b023f --- /dev/null +++ b/Properties/Resources.resx @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + ..\resources\icon.png;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + ..\resources\icon.ico;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c2f71e4 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# Firefly Database + +Firefly is a high-performance, Redis-compatible key-value store optimized for gaming applications. Built with native AOT compilation support for maximum performance. + +## Features + +- **Redis-Compatible**: Uses Redis Serialization Protocol (RESP) for compatibility with Redis clients +- **Concurrent Data Structures**: Uses parallel data structures for high throughput and low latency +- **Sharded Architecture**: Distributes data across multiple shards to reduce contention +- **Password Authentication**: Optional password protection for secure deployments +- **Native AOT Support**: Fully compatible with .NET Native AOT for maximum performance +- **Automatic Backups**: Configurable backup system with rotation +- **Multi-Datatype Support**: Strings, Lists and Hash Maps support + +## Quick Start + +```bash +# Start server with default settings +firefly + +# Start with password authentication +firefly --password yourSecretPassword + +# Start with custom port +firefly --port 6380 +``` + +## Authentication + +When starting Firefly with password protection, clients must authenticate: + +``` +AUTH yourSecretPassword +``` + +For more details on how to use authentication with different clients, see the [USAGE.md](USAGE.md) file. + +## Building from Source + +### Prerequisites +- .NET 9.0 SDK or later + +### Build Commands +```bash +# Standard build +dotnet build + +# Release build +dotnet build -c Release + +# Native AOT build (Windows x64) +dotnet publish -c Release -r win-x64 --self-contained -p:PublishAot=true + +# Native AOT build (Linux x64) +dotnet publish -c Release -r linux-x64 --self-contained -p:PublishAot=true +``` + +## Performance Benchmarks + +The benchmark mode (`--benchmark`) runs a series of concurrent operations to test throughput: + +```bash +firefly --benchmark +``` + +Typical results on modern hardware: +- ~500,000 operations per second for simple key-value operations +- ~250,000 operations per second for list operations +- ~150,000 operations per second for hash map operations + +## License + +MIT + +## See Also + +- [USAGE.md](USAGE.md) - Detailed usage instructions +- [ArmaFireflyClient](https://gitea.innovativedevsolutions.org/IDSolutions/ArmaFireflyClient/) - Arma 3 extension client \ No newline at end of file diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..afda55c --- /dev/null +++ b/USAGE.md @@ -0,0 +1,167 @@ +# Firefly Server Usage Guide + +Firefly is key-value store optimized for performance with native AOT compilation support. + +## Table of Contents +- [Starting the Server](#starting-the-server) +- [Password Authentication](#password-authentication) +- [Basic Operations](#basic-operations) +- [Client Integration](#client-integration) +- [ArmaFireflyClient Integration](#armafireflyclient-integration) +- [Performance Considerations](#performance-considerations) + +## Starting the Server + +### Basic Start +```bash +# Start with default settings (port 6379, backups enabled) +firefly +``` + +### With Command Line Options +```bash +# Start with custom port +firefly --port 6380 + +# Start with authentication enabled +firefly --password yourSecretPassword + +# Start with backups disabled +firefly --no-backup + +# Start with custom backup interval (10 minutes) +firefly --backup-interval 10 + +# Start with limited backup files +firefly --max-backups 5 + +# Start with shorter connection timeout +firefly --timeout 60 +``` + +### Full Command Reference +Run `firefly --help` to see all available options. + +## Password Authentication + +When starting Firefly with a password, clients must authenticate before running commands: + +``` +AUTH yourSecretPassword +``` + +### Authentication Behaviors + +- If no password is set, all commands work without authentication +- If a password is set, only PING and AUTH commands work without authentication +- After successful authentication, all commands work normally +- Failed authentication attempts return an error but allow retries + +## Basic Operations + +### String Operations +``` +SET key value # Set a key-value pair +GET key # Get a value by key +DEL key # Delete a key (works for all data types) +``` + +### List Operations +``` +LPUSH key value1 value2 # Add values to the beginning of a list +RPUSH key value1 value2 # Add values to the end of a list +LPOP key # Remove and return the first value +RPOP key # Remove and return the last value +LRANGE key start stop # Get a range of values (inclusive) +LINDEX key index # Get a value at a specific index +``` + +### Hash Table Operations +``` +HSET key field value # Set a field in a hash +HGET key field # Get a field from a hash +HDEL key field # Delete a field from a hash +HEXISTS key field # Check if field exists in hash +HGETALL key # Get all fields and values +HMSET key field1 value1 field2 value2 # Set multiple fields at once +``` + +### Backup Operations +``` +SAVE # Save data immediately +BGSAVE # Save data in the background +``` + +## Client Integration + +### Connecting with Redis Client +Firefly is compatible with Redis protocol. Use your preferred Redis client library: + +```csharp +// C# example using StackExchange.Redis +ConnectionMultiplexer redis = ConnectionMultiplexer.Connect("localhost:6379,password=yourpassword"); +IDatabase db = redis.GetDatabase(); +db.StringSet("mykey", "myvalue"); +string value = db.StringGet("mykey"); +``` + +### Connecting with Raw TCP +```csharp +// Basic TCP connection example +using System.Net.Sockets; +using System.Text; + +TcpClient client = new TcpClient("localhost", 6379); +NetworkStream stream = client.GetStream(); + +// Authenticate if needed +byte[] authCmd = Encoding.UTF8.GetBytes("AUTH yourpassword\r\n"); +await stream.WriteAsync(authCmd); +// ... read response ... + +// Send command +byte[] setCmd = Encoding.UTF8.GetBytes("SET mykey myvalue\r\n"); +await stream.WriteAsync(setCmd); +// ... read response ... +``` + +## ArmaFireflyClient Integration + +When using ArmaFireflyClient with Arma 3, you need to ensure the client has the correct password configured. + +### Setting Up Connection in Arma 3 +```sqf +// Connect to a password-protected Firefly server +"setup" callExtension ["127.0.0.1", "6379", "yourSecretPassword"]; +``` + +This updates the client configuration to use the specified password for authentication. The client will automatically authenticate when connecting to the server. + +### Using Default XML Configuration +Create a config.xml file in your @firefly directory: +```xml + + + 127.0.0.1 + 6379 + yourSecretPassword + false + false + +``` + +### Warning About Empty Password +Setting up ArmaFireflyClient without a password when connecting to a password-protected Firefly server will result in authentication failures. Always make sure your password matches the server's password. + +## Performance Considerations + +- Password authentication adds a small overhead to the initial connection +- For best performance with authenticated connections, reuse existing connections +- The server uses concurrent data structures to maintain performance even with authentication enabled +- Consider using a longer connection timeout (`--timeout`) to keep connections alive for longer periods + +## Common Issues + +1. **Authentication Errors**: Make sure the client is using the correct password +2. **Connection Refused**: Verify the server is running and listening on the expected port +3. **Timeout Errors**: Increase the connection timeout or check for network issues \ No newline at end of file diff --git a/artifacts/exe/Firefly b/artifacts/exe/Firefly new file mode 100644 index 0000000..9d30881 Binary files /dev/null and b/artifacts/exe/Firefly differ diff --git a/artifacts/exe/Firefly.dbg b/artifacts/exe/Firefly.dbg new file mode 100644 index 0000000..3c03272 Binary files /dev/null and b/artifacts/exe/Firefly.dbg differ diff --git a/artifacts/exe/Firefly.exe b/artifacts/exe/Firefly.exe new file mode 100644 index 0000000..c3a617b Binary files /dev/null and b/artifacts/exe/Firefly.exe differ diff --git a/artifacts/exe/Firefly.pdb b/artifacts/exe/Firefly.pdb new file mode 100644 index 0000000..dc4b1d0 Binary files /dev/null and b/artifacts/exe/Firefly.pdb differ diff --git a/artifacts/exe/Firefly.xml b/artifacts/exe/Firefly.xml new file mode 100644 index 0000000..fd2e5e4 --- /dev/null +++ b/artifacts/exe/Firefly.xml @@ -0,0 +1,677 @@ + + + + Firefly + + + + + A strongly-typed resource class, for looking up localized strings, etc. + + + + + Returns the cached ResourceManager instance used by this class. + + + + + Overrides the current thread's CurrentUICulture property for all + resource lookups using this strongly typed resource class. + + + + + Looks up a localized resource of type System.Byte[]. + + + + + Looks up a localized resource of type System.Byte[]. + + + + + Core class implementing the Firefly database server functionality + + + + + Handles the HSET command which sets a field in a hash. + + Command arguments in format: "key field value" + 1 if the field was added, 0 if it was updated + + + + Handles the HGET command which retrieves the value of a field in a hash. + + Command arguments in format: "key field" + The value of the field, or nil if the field doesn't exist + + + + Handles the HDEL command which removes a field from a hash. + + Command arguments in format: "key field" + 1 if the field was removed, 0 if it didn't exist + + + + Handles the HEXISTS command which checks if a field exists in a hash. + + Command arguments in format: "key field" + 1 if the field exists, 0 if it doesn't + + + + Handles the HGETALL command which retrieves all fields and values in a hash. + + Command arguments in format: "key" + All fields and values in the hash, or nil if the hash doesn't exist + + + + Handles the HMSET command which sets multiple fields in a hash. + + Command arguments in format: "key field value [field value ...]" + OK on success, error if arguments are invalid + + + + Gets or creates a hash for a given key. + + The key to get or create the hash for + The hash + + + + Checks if a hash exists for a given key. + + The key to check the hash for + True if the hash exists, false otherwise + + + + Gets a hash for a given key. + + The key to get the hash for + The hash + True if the hash was found, false otherwise + + + + Removes a hash for a given key. + + The key to remove the hash for + True if the hash was removed, false otherwise + + + + Splits a string respecting quoted sections. + + The input string to split + An array of tokens from the input string + + + + Pattern:
+ ([^\\s"]+)|"([^"]*)"
+ Explanation:
+ + ○ Match with 2 alternative expressions, atomically.
+ ○ 1st capture group.
+ ○ Match a character in the set [^"\s] atomically at least once.
+ ○ Match a sequence of expressions.
+ ○ Match '"'.
+ ○ 2nd capture group.
+ ○ Match a character other than '"' atomically any number of times.
+ ○ Match '"'.
+ + + + + + Creates an array of shards of type T. + + The type of the shards + An array of shards + + + + Creates an array of ReaderWriterLockSlim instances for list operations. + + An array of locks + + + + Gets the shard index for a given key. + + The key to get the shard index for + The shard index + + + + Creates an array of locks. + + An array of locks + + + + Checks if a key exists in any store (string, list, or hash). + + The key to check + True if the key exists in any store, false otherwise + + + + Gets the type of a key if it exists in any store. + + The key to check + The type of the key ("string", "list", "hash", or null if not found) + + + + Ensures a key doesn't exist in any store before creating it in the target store. + + The key to check + The type of store where the key will be created + True if the key can be created, false if it already exists with a different type + + + + Handles the DEL command which removes a key from all stores (string, list, hash). + + The key to delete + The number of keys that were removed + + + + Handles the TYPE command which returns the type of a key. + + The key to check the type of + The type of the key as a string response + + + + Handles the KEYS command which returns all keys matching a pattern. + + The pattern to match + A list of keys matching the pattern + + + + Handles the LPUSH command which adds an element to the left of a list. + + Command arguments in format: "key value" + The length of the list after the push operation + + + + Handles the RPUSH command which adds values to the tail of a list. + + Command arguments in format: "key value1 [value2 ...]" + Response indicating the new length of the list + + + + Handles the LPOP command which removes and returns the first element of a list. + + Command arguments containing the key + The popped value or nil if the list is empty + + + + Handles the RPOP command which removes and returns the last element of a list. + + Command arguments containing the key + The popped value or nil if the list is empty + + + + Handles the LRANGE command which returns a range of elements from a list. + + Command arguments in format: "key start stop" + Array of elements in the specified range + + + + Handles the LINDEX command which returns an element from a list by its index. + + Command arguments in format: "key index" + The element at the specified index or nil if not found + + + + Handles the LSET command which sets the value of an element in a list by its index. + + Command arguments in format: "key index value" + OK on success, error if index is out of range + + + + Handles the LPOS command which returns the position of an element in a list. + + Command arguments in format: "key element [RANK rank] [MAXLEN len]" + The position of the element or nil if not found + + + + Handles the LTRIM command which trims a list to the specified range. + + Command arguments in format: "key start stop" + OK on success, error if arguments are invalid + + + + Handles the LREM command which removes elements equal to the given value from a list. + + Command arguments in format: "key count element" + The number of removed elements + + + + Gets or creates a list for a given key. + + The key to get or create the list for + The list + + + + Checks if a list exists for a given key. + + The key to check the list for + True if the list exists, false otherwise + + + + Gets a list for a given key. + + The key to get the list for + The list + True if the list was found, false otherwise + + + + Removes a list for a given key. + + The key to remove the list for + True if the list was removed, false otherwise + + + + Executes an action with a write lock on a list for a given key. + + The key to execute the action on + The action to execute + + + + Executes an action with a read lock on a list for a given key. + + The type of the result + The key to execute the action on + The action to execute + + + + Handles the AUTH command which authenticates a client. + + The password to authenticate with + The client ID + OK on success, error if authentication fails + + + + Checks if a client is authenticated. + + The client ID + True if the client is authenticated, false otherwise + + + + Handles the SAVE command which triggers a manual backup of the data. + + + + + Handles the BGSAVE command which triggers an asynchronous backup of the data. + + + + + Handles server shutdown operations + + + + + Handles the SET command which sets a key-value pair in the string store. + + Command arguments in format: "key value" + OK on success, error if arguments are invalid + + + + Handles the GET command which retrieves a value from the string store. + + Command arguments containing the key + The value associated with the key, or nil if the key doesn't exist + + + + Checks if a string exists for a given key. + + The key to check the string for + True if the string exists, false otherwise + + + + Sets a string for a given key. + + The key to set the string for + The value to set + True if the string was set, false otherwise + + + + Gets a string for a given key. + + The key to get the string for + The value + True if the string was found, false otherwise + + + + Removes a string for a given key. + + The key to remove the string for + True if the string was removed, false otherwise + + + + Container class for all Firefly database data used in serialization and backup operations. + + + + + Dictionary containing all string key-value pairs stored in the database. + + + + + Dictionary containing all lists stored in the database. + + + + + Dictionary containing all hash tables stored in the database. + + + + + Timestamp when the backup was created. + + + + + Test client for verifying pipelining and batching functionality + + + + + Runs the pipeline test + + + + + Read responses from the stream until we have the expected count + + + + + Test with sequential commands (no pipelining) + + + + + Test with pipelined commands + + + + + Test with batched commands + + + + + Defines the Firefly wire protocol specification for 3rd party clients + + + + + The current version of the Firefly protocol + + + + + Prefix for simple string responses (+) + + + + + Prefix for error responses (-) + + + + + Prefix for integer responses (:) + + + + + Prefix for bulk string responses ($) + + + + + Prefix for array responses (*) + + + + + Special message indicating a nil value ($-1) + + + + + Standard message terminator sequence (\r\n) + + + + + Standard OK response (+OK\r\n) + + + + + Response for queued pipeline commands (+QUEUED\r\n) + + + + + Standard nil response ($-1\r\n) + + + + + Command format specification for external clients + + + + + Authenticate with the server using a password + + + + + Test server connection + + + + + Close the connection + + + + + Set a key to hold a string value + + + + + Get the value of a key + + + + + Delete a key + + + + + Insert elements at the head of a list + + + + + Insert elements at the tail of a list + + + + + Remove and get the first element in a list + + + + + Remove and get the last element in a list + + + + + Get a range of elements from a list + + + + + Set the string value of a hash field + + + + + Get the value of a hash field + + + + + Delete a hash field + + + + + Get all fields and values in a hash + + + + + Start a pipeline for batch processing + + + + + Execute all commands in the pipeline + + + + + Response format specification for external clients + + + + + Format a simple string response + + The string value to format + Formatted string response + + + + Format an error response + + The error message + Formatted error response + + + + Format an integer response + + The integer value to format + Formatted integer response + + + + Format a bulk string response + + The string value to format + Formatted bulk string response + + + + Format an array response + + Array of strings to format + Formatted array response + + + Custom -derived type for the MyRegex method. + + + Cached, thread-safe singleton instance. + + + Initializes the instance. + + + Provides a factory for creating instances to be used by methods on . + + + Creates an instance of a used by methods on . + + + Provides the runner that contains the custom logic implementing the specified regular expression. + + + Scan the starting from base.runtextstart for the next match. + The text being scanned by the regular expression. + + + Search starting from base.runtextpos for the next location a match could possibly start. + The text being scanned by the regular expression. + true if a possible match was found; false if no more matches are possible. + + + Determine whether at base.runtextpos is a match for the regular expression. + The text being scanned by the regular expression. + true if the regular expression matches at the current position; otherwise, false. + + + Helper methods used by generated -derived implementations. + + + Default timeout value set in , or if none was set. + + + Whether is non-infinite. + + + diff --git a/artifacts/native/Firefly.dll b/artifacts/native/Firefly.dll new file mode 100644 index 0000000..af6f26b Binary files /dev/null and b/artifacts/native/Firefly.dll differ diff --git a/artifacts/native/Firefly.pdb b/artifacts/native/Firefly.pdb new file mode 100644 index 0000000..1d98a73 Binary files /dev/null and b/artifacts/native/Firefly.pdb differ diff --git a/artifacts/native/Firefly.so b/artifacts/native/Firefly.so new file mode 100644 index 0000000..bfbd90c Binary files /dev/null and b/artifacts/native/Firefly.so differ diff --git a/artifacts/native/Firefly.so.dbg b/artifacts/native/Firefly.so.dbg new file mode 100644 index 0000000..4fda6ad Binary files /dev/null and b/artifacts/native/Firefly.so.dbg differ diff --git a/artifacts/native/Firefly.xml b/artifacts/native/Firefly.xml new file mode 100644 index 0000000..fd2e5e4 --- /dev/null +++ b/artifacts/native/Firefly.xml @@ -0,0 +1,677 @@ + + + + Firefly + + + + + A strongly-typed resource class, for looking up localized strings, etc. + + + + + Returns the cached ResourceManager instance used by this class. + + + + + Overrides the current thread's CurrentUICulture property for all + resource lookups using this strongly typed resource class. + + + + + Looks up a localized resource of type System.Byte[]. + + + + + Looks up a localized resource of type System.Byte[]. + + + + + Core class implementing the Firefly database server functionality + + + + + Handles the HSET command which sets a field in a hash. + + Command arguments in format: "key field value" + 1 if the field was added, 0 if it was updated + + + + Handles the HGET command which retrieves the value of a field in a hash. + + Command arguments in format: "key field" + The value of the field, or nil if the field doesn't exist + + + + Handles the HDEL command which removes a field from a hash. + + Command arguments in format: "key field" + 1 if the field was removed, 0 if it didn't exist + + + + Handles the HEXISTS command which checks if a field exists in a hash. + + Command arguments in format: "key field" + 1 if the field exists, 0 if it doesn't + + + + Handles the HGETALL command which retrieves all fields and values in a hash. + + Command arguments in format: "key" + All fields and values in the hash, or nil if the hash doesn't exist + + + + Handles the HMSET command which sets multiple fields in a hash. + + Command arguments in format: "key field value [field value ...]" + OK on success, error if arguments are invalid + + + + Gets or creates a hash for a given key. + + The key to get or create the hash for + The hash + + + + Checks if a hash exists for a given key. + + The key to check the hash for + True if the hash exists, false otherwise + + + + Gets a hash for a given key. + + The key to get the hash for + The hash + True if the hash was found, false otherwise + + + + Removes a hash for a given key. + + The key to remove the hash for + True if the hash was removed, false otherwise + + + + Splits a string respecting quoted sections. + + The input string to split + An array of tokens from the input string + + + + Pattern:
+ ([^\\s"]+)|"([^"]*)"
+ Explanation:
+ + ○ Match with 2 alternative expressions, atomically.
+ ○ 1st capture group.
+ ○ Match a character in the set [^"\s] atomically at least once.
+ ○ Match a sequence of expressions.
+ ○ Match '"'.
+ ○ 2nd capture group.
+ ○ Match a character other than '"' atomically any number of times.
+ ○ Match '"'.
+ + + + + + Creates an array of shards of type T. + + The type of the shards + An array of shards + + + + Creates an array of ReaderWriterLockSlim instances for list operations. + + An array of locks + + + + Gets the shard index for a given key. + + The key to get the shard index for + The shard index + + + + Creates an array of locks. + + An array of locks + + + + Checks if a key exists in any store (string, list, or hash). + + The key to check + True if the key exists in any store, false otherwise + + + + Gets the type of a key if it exists in any store. + + The key to check + The type of the key ("string", "list", "hash", or null if not found) + + + + Ensures a key doesn't exist in any store before creating it in the target store. + + The key to check + The type of store where the key will be created + True if the key can be created, false if it already exists with a different type + + + + Handles the DEL command which removes a key from all stores (string, list, hash). + + The key to delete + The number of keys that were removed + + + + Handles the TYPE command which returns the type of a key. + + The key to check the type of + The type of the key as a string response + + + + Handles the KEYS command which returns all keys matching a pattern. + + The pattern to match + A list of keys matching the pattern + + + + Handles the LPUSH command which adds an element to the left of a list. + + Command arguments in format: "key value" + The length of the list after the push operation + + + + Handles the RPUSH command which adds values to the tail of a list. + + Command arguments in format: "key value1 [value2 ...]" + Response indicating the new length of the list + + + + Handles the LPOP command which removes and returns the first element of a list. + + Command arguments containing the key + The popped value or nil if the list is empty + + + + Handles the RPOP command which removes and returns the last element of a list. + + Command arguments containing the key + The popped value or nil if the list is empty + + + + Handles the LRANGE command which returns a range of elements from a list. + + Command arguments in format: "key start stop" + Array of elements in the specified range + + + + Handles the LINDEX command which returns an element from a list by its index. + + Command arguments in format: "key index" + The element at the specified index or nil if not found + + + + Handles the LSET command which sets the value of an element in a list by its index. + + Command arguments in format: "key index value" + OK on success, error if index is out of range + + + + Handles the LPOS command which returns the position of an element in a list. + + Command arguments in format: "key element [RANK rank] [MAXLEN len]" + The position of the element or nil if not found + + + + Handles the LTRIM command which trims a list to the specified range. + + Command arguments in format: "key start stop" + OK on success, error if arguments are invalid + + + + Handles the LREM command which removes elements equal to the given value from a list. + + Command arguments in format: "key count element" + The number of removed elements + + + + Gets or creates a list for a given key. + + The key to get or create the list for + The list + + + + Checks if a list exists for a given key. + + The key to check the list for + True if the list exists, false otherwise + + + + Gets a list for a given key. + + The key to get the list for + The list + True if the list was found, false otherwise + + + + Removes a list for a given key. + + The key to remove the list for + True if the list was removed, false otherwise + + + + Executes an action with a write lock on a list for a given key. + + The key to execute the action on + The action to execute + + + + Executes an action with a read lock on a list for a given key. + + The type of the result + The key to execute the action on + The action to execute + + + + Handles the AUTH command which authenticates a client. + + The password to authenticate with + The client ID + OK on success, error if authentication fails + + + + Checks if a client is authenticated. + + The client ID + True if the client is authenticated, false otherwise + + + + Handles the SAVE command which triggers a manual backup of the data. + + + + + Handles the BGSAVE command which triggers an asynchronous backup of the data. + + + + + Handles server shutdown operations + + + + + Handles the SET command which sets a key-value pair in the string store. + + Command arguments in format: "key value" + OK on success, error if arguments are invalid + + + + Handles the GET command which retrieves a value from the string store. + + Command arguments containing the key + The value associated with the key, or nil if the key doesn't exist + + + + Checks if a string exists for a given key. + + The key to check the string for + True if the string exists, false otherwise + + + + Sets a string for a given key. + + The key to set the string for + The value to set + True if the string was set, false otherwise + + + + Gets a string for a given key. + + The key to get the string for + The value + True if the string was found, false otherwise + + + + Removes a string for a given key. + + The key to remove the string for + True if the string was removed, false otherwise + + + + Container class for all Firefly database data used in serialization and backup operations. + + + + + Dictionary containing all string key-value pairs stored in the database. + + + + + Dictionary containing all lists stored in the database. + + + + + Dictionary containing all hash tables stored in the database. + + + + + Timestamp when the backup was created. + + + + + Test client for verifying pipelining and batching functionality + + + + + Runs the pipeline test + + + + + Read responses from the stream until we have the expected count + + + + + Test with sequential commands (no pipelining) + + + + + Test with pipelined commands + + + + + Test with batched commands + + + + + Defines the Firefly wire protocol specification for 3rd party clients + + + + + The current version of the Firefly protocol + + + + + Prefix for simple string responses (+) + + + + + Prefix for error responses (-) + + + + + Prefix for integer responses (:) + + + + + Prefix for bulk string responses ($) + + + + + Prefix for array responses (*) + + + + + Special message indicating a nil value ($-1) + + + + + Standard message terminator sequence (\r\n) + + + + + Standard OK response (+OK\r\n) + + + + + Response for queued pipeline commands (+QUEUED\r\n) + + + + + Standard nil response ($-1\r\n) + + + + + Command format specification for external clients + + + + + Authenticate with the server using a password + + + + + Test server connection + + + + + Close the connection + + + + + Set a key to hold a string value + + + + + Get the value of a key + + + + + Delete a key + + + + + Insert elements at the head of a list + + + + + Insert elements at the tail of a list + + + + + Remove and get the first element in a list + + + + + Remove and get the last element in a list + + + + + Get a range of elements from a list + + + + + Set the string value of a hash field + + + + + Get the value of a hash field + + + + + Delete a hash field + + + + + Get all fields and values in a hash + + + + + Start a pipeline for batch processing + + + + + Execute all commands in the pipeline + + + + + Response format specification for external clients + + + + + Format a simple string response + + The string value to format + Formatted string response + + + + Format an error response + + The error message + Formatted error response + + + + Format an integer response + + The integer value to format + Formatted integer response + + + + Format a bulk string response + + The string value to format + Formatted bulk string response + + + + Format an array response + + Array of strings to format + Formatted array response + + + Custom -derived type for the MyRegex method. + + + Cached, thread-safe singleton instance. + + + Initializes the instance. + + + Provides a factory for creating instances to be used by methods on . + + + Creates an instance of a used by methods on . + + + Provides the runner that contains the custom logic implementing the specified regular expression. + + + Scan the starting from base.runtextstart for the next match. + The text being scanned by the regular expression. + + + Search starting from base.runtextpos for the next location a match could possibly start. + The text being scanned by the regular expression. + true if a possible match was found; false if no more matches are possible. + + + Determine whether at base.runtextpos is a match for the regular expression. + The text being scanned by the regular expression. + true if the regular expression matches at the current position; otherwise, false. + + + Helper methods used by generated -derived implementations. + + + Default timeout value set in , or if none was set. + + + Whether is non-infinite. + + + diff --git a/build-all.ps1 b/build-all.ps1 new file mode 100644 index 0000000..335b64b --- /dev/null +++ b/build-all.ps1 @@ -0,0 +1,44 @@ +# Build configuration +$configuration = "Release" +$baseOutputPath = ".\artifacts" + +# Determine current OS and platform +$currentPlatform = "win-x64" # Since we're running PowerShell, we're on Windows +Write-Host "Building for current platform: $currentPlatform" + +# Create output directories +$exeOutputPath = "$baseOutputPath\exe" +$nativeOutputPath = "$baseOutputPath\native" +New-Item -ItemType Directory -Force -Path $exeOutputPath | Out-Null +New-Item -ItemType Directory -Force -Path $nativeOutputPath | Out-Null + +# Build executable +Write-Host "Building executable..." +dotnet publish ` + -c $configuration ` + -r $currentPlatform ` + --self-contained true ` + -p:PublishDir=$exeOutputPath + +if ($LASTEXITCODE -ne 0) { + Write-Host "Executable build failed. Check the error messages above." + exit 1 +} + +# Build native shared library +Write-Host "Building native shared library..." +dotnet publish ` + -c $configuration ` + -r $currentPlatform ` + --self-contained true ` + -p:BuildType=lib ` + -p:PublishDir=$nativeOutputPath + +if ($LASTEXITCODE -ne 0) { + Write-Host "Library build failed. Check the error messages above." + exit 1 +} + +Write-Host "Build complete. Outputs available at:" +Write-Host "Executable: $exeOutputPath" +Write-Host "Native library: $nativeOutputPath" \ No newline at end of file diff --git a/build-all.sh b/build-all.sh new file mode 100644 index 0000000..67e661c --- /dev/null +++ b/build-all.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +configuration="Release" +baseOutputPath="./artifacts" + +# Detect platform +if [[ "$OSTYPE" == "darwin"* ]]; then + currentPlatform="osx-x64" +else + currentPlatform="linux-x64" +fi + +echo "Building for current platform: $currentPlatform" + +# Create output directories +exeOutputPath="$baseOutputPath/exe" +nativeOutputPath="$baseOutputPath/native" +mkdir -p "$exeOutputPath" +mkdir -p "$nativeOutputPath" + +# Build executable +echo "Building executable..." +dotnet publish \ + -c $configuration \ + -r $currentPlatform \ + --self-contained true \ + -p:PublishDir=$exeOutputPath + +if [ $? -ne 0 ]; then + echo "Executable build failed. Check the error messages above." + exit 1 +fi + +# Build native shared library +echo "Building native shared library..." +dotnet publish \ + -c $configuration \ + -r $currentPlatform \ + --self-contained true \ + -p:BuildType=lib \ + -p:PublishDir=$nativeOutputPath + +if [ $? -ne 0 ]; then + echo "Library build failed. Check the error messages above." + exit 1 +fi + +echo "Build complete. Outputs available at:" +echo "Executable: $exeOutputPath" +echo "Native library: $nativeOutputPath" + +# Make the outputs executable +if [[ "$currentPlatform" == "linux-x64" ]]; then + chmod +x "$exeOutputPath/Firefly" + chmod +x "$nativeOutputPath/Firefly.so" +elif [[ "$currentPlatform" == "osx-x64" ]]; then + chmod +x "$exeOutputPath/Firefly" + chmod +x "$nativeOutputPath/Firefly.dylib" +fi \ No newline at end of file diff --git a/icon.ico b/icon.ico new file mode 100644 index 0000000..accd97b Binary files /dev/null and b/icon.ico differ diff --git a/icon.png b/icon.png new file mode 100644 index 0000000..1d5a019 Binary files /dev/null and b/icon.png differ diff --git a/resources/icon.ico b/resources/icon.ico new file mode 100644 index 0000000..accd97b Binary files /dev/null and b/resources/icon.ico differ diff --git a/resources/icon.png b/resources/icon.png new file mode 100644 index 0000000..1d5a019 Binary files /dev/null and b/resources/icon.png differ diff --git a/src/BackupSystem.cs b/src/BackupSystem.cs new file mode 100644 index 0000000..f4adf8d --- /dev/null +++ b/src/BackupSystem.cs @@ -0,0 +1,358 @@ +using System.Collections.Concurrent; + +namespace Firefly +{ + /// + /// Core class implementing the Firefly database server functionality + /// + public partial class Firefly + { + #region Backup System + static void InitializeBackupSystem() + { + // Create backup directory if it doesn't exist + if (!Directory.Exists(backupDirectory)) + { + try + { + Directory.CreateDirectory(backupDirectory); + Console.WriteLine($"Created backup directory: {backupDirectory}"); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to create backup directory: {ex.Message}"); + Console.WriteLine("Backups will be disabled"); + backupsEnabled = false; + return; + } + } + + // Set up automatic backup timer + backupTimer = new System.Timers.Timer(autoBackupIntervalMinutes * 60 * 1000); // Convert minutes to milliseconds + backupTimer.Elapsed += (sender, e) => BackupData(); + backupTimer.AutoReset = true; + backupTimer.Start(); + + Console.WriteLine($"Automatic backup system initialized (every {autoBackupIntervalMinutes} minutes)"); + Console.WriteLine($"Keeping up to {maxBackupFiles} backup files"); + + // Register backup on exit + AppDomain.CurrentDomain.ProcessExit += (sender, e) => + { + if (backupsEnabled) + { + Console.WriteLine("Server shutting down, performing final backup..."); + BackupData(); + } + }; + } + + static void LoadDataFromBackup() + { + if (!backupsEnabled) + { + Console.WriteLine("Backups are disabled, starting with empty database"); + return; + } + + string? mostRecentBackup = GetMostRecentBackupFile(); + if (mostRecentBackup != null) + { + Console.WriteLine($"Loading data from backup: {Path.GetFileName(mostRecentBackup)}"); + try + { + using var fileStream = new FileStream(mostRecentBackup, FileMode.Open, FileAccess.Read); + using var binaryReader = new BinaryReader(fileStream); + + // Read backup time + long backupTimeBinary = binaryReader.ReadInt64(); + DateTime backupTime = DateTime.FromBinary(backupTimeBinary); + Console.WriteLine($"Backup was created on: {backupTime:f}"); + + // Read string store + int stringCount = binaryReader.ReadInt32(); + Console.WriteLine($"Loading {stringCount} string keys from backup"); + for (int i = 0; i < stringCount; i++) + { + string key = binaryReader.ReadString(); + string value = binaryReader.ReadString(); + int shardIndex = GetShardIndex(key); + stringStoreShards[shardIndex][key] = value; + } + Console.WriteLine($"Loaded {stringCount} string keys from backup"); + + // Read list store + int listCount = binaryReader.ReadInt32(); + Console.WriteLine($"Loading {listCount} list keys from backup"); + for (int i = 0; i < listCount; i++) + { + string key = binaryReader.ReadString(); + int itemCount = binaryReader.ReadInt32(); + var list = new List(); + + for (int j = 0; j < itemCount; j++) + { + string item = binaryReader.ReadString(); + list.Add(item); + } + + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterWriteLock(); + try + { + listStoreShards[shardIndex][key] = list; + } + finally + { + listStoreLocks[shardIndex].ExitWriteLock(); + } + } + Console.WriteLine($"Loaded {listCount} list keys from backup"); + + // Read hash store + int hashCount = binaryReader.ReadInt32(); + Console.WriteLine($"Loading {hashCount} hash keys from backup"); + for (int i = 0; i < hashCount; i++) + { + string key = binaryReader.ReadString(); + int fieldCount = binaryReader.ReadInt32(); + var concurrentHash = new ConcurrentDictionary(); + + for (int j = 0; j < fieldCount; j++) + { + string fieldKey = binaryReader.ReadString(); + string fieldValue = binaryReader.ReadString(); + concurrentHash[fieldKey] = fieldValue; + } + + int shardIndex = GetShardIndex(key); + hashStoreShards[shardIndex][key] = concurrentHash; + } + Console.WriteLine($"Loaded {hashCount} hash keys from backup"); + + Console.WriteLine($"Data successfully loaded from backup: {Path.GetFileName(mostRecentBackup)}"); + Console.WriteLine($"Total keys loaded: {stringCount + listCount + hashCount}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error loading data from backup: {ex.Message}"); + Console.WriteLine($"Stack trace: {ex.StackTrace}"); + Console.WriteLine("The backup file may be corrupted or created with an incompatible version"); + Console.WriteLine("Starting with empty database"); + } + } + else + { + Console.WriteLine("No backup files found, starting with empty database"); + } + } + + // Get the most recent backup file + static string? GetMostRecentBackupFile() + { + if (!Directory.Exists(backupDirectory)) + return null; + + try + { + var backupFiles = Directory.GetFiles(backupDirectory, backupFilePrefix + "*" + backupFileExtension) + .OrderByDescending(file => File.GetLastWriteTime(file)) + .ToArray(); + + return backupFiles.Length > 0 ? backupFiles[0] : null; + } + catch (Exception ex) + { + Console.WriteLine($"Error accessing backup files: {ex.Message}"); + return null; + } + } + + // Deletes oldest backup files if the number exceeds maxBackupFiles + static void RotateBackupFiles() + { + if (!Directory.Exists(backupDirectory)) + return; + + try + { + var backupFiles = Directory.GetFiles(backupDirectory, backupFilePrefix + "*" + backupFileExtension) + .OrderByDescending(file => File.GetLastWriteTime(file)) + .ToArray(); + + // Remove oldest files if we have more than maxBackupFiles + if (backupFiles.Length > maxBackupFiles) + { + for (int i = maxBackupFiles; i < backupFiles.Length; i++) + { + try + { + File.Delete(backupFiles[i]); + Console.WriteLine($"Deleted old backup file: {Path.GetFileName(backupFiles[i])}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error deleting old backup file {backupFiles[i]}: {ex.Message}"); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error during backup rotation: {ex.Message}"); + } + } + + static void BackupData() + { + // Skip if backups are disabled + if (!backupsEnabled) + return; + + // Use a global lock to prevent backup conflicts + backupLock.Enter(() => + { + try + { + var data = new FireflyData + { + BackupTime = DateTime.Now + }; + + // Copy string store from all shards + for (int i = 0; i < ShardCount; i++) + { + foreach (var kvp in stringStoreShards[i]) + { + data.StringStore[kvp.Key] = kvp.Value; + } + } + + // Copy list store from all shards + for (int i = 0; i < ShardCount; i++) + { + listStoreLocks[i].EnterReadLock(); + try + { + foreach (var kvp in listStoreShards[i]) + { + data.ListStore[kvp.Key] = [.. kvp.Value]; + } + } + finally + { + listStoreLocks[i].ExitReadLock(); + } + } + + // Copy hash store from all shards + for (int i = 0; i < ShardCount; i++) + { + foreach (var kvp in hashStoreShards[i]) + { + // Convert ConcurrentDictionary to Dictionary for serialization + var regularDict = new Dictionary(); + foreach (var fieldKvp in kvp.Value) + { + regularDict[fieldKvp.Key] = fieldKvp.Value; + } + data.HashStore[kvp.Key] = regularDict; + } + } + + // Check if we have any data to back up + if (data.StringStore.Count > 0 || data.ListStore.Count > 0 || data.HashStore.Count > 0) + { + // Generate timestamp for the filename + string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); + string backupFileName = $"{backupFilePrefix}_{timestamp}{backupFileExtension}"; + string backupFilePath = Path.Combine(backupDirectory, backupFileName); + + try + { + // Serialize and save data - use a temporary file first to avoid corruption + string tempFilePath = backupFilePath + ".tmp"; + + // Replace JSON serialization with BinaryFormatter + using (var fileStream = new FileStream(tempFilePath, FileMode.Create)) + { + using var binaryWriter = new BinaryWriter(fileStream); + // Write backup time + binaryWriter.Write(data.BackupTime.ToBinary()); + + // Write string store + binaryWriter.Write(data.StringStore.Count); + foreach (var kvp in data.StringStore) + { + binaryWriter.Write(kvp.Key); + binaryWriter.Write(kvp.Value); + } + + // Write list store + binaryWriter.Write(data.ListStore.Count); + foreach (var kvp in data.ListStore) + { + binaryWriter.Write(kvp.Key); + binaryWriter.Write(kvp.Value.Count); + foreach (var item in kvp.Value) + { + binaryWriter.Write(item); + } + } + + // Write hash store + binaryWriter.Write(data.HashStore.Count); + foreach (var kvp in data.HashStore) + { + binaryWriter.Write(kvp.Key); + binaryWriter.Write(kvp.Value.Count); + foreach (var fieldKvp in kvp.Value) + { + binaryWriter.Write(fieldKvp.Key); + binaryWriter.Write(fieldKvp.Value); + } + } + } + + // If successfully written, move to final location + if (File.Exists(tempFilePath)) + { + // Delete the target file if it already exists + if (File.Exists(backupFilePath)) + { + File.Delete(backupFilePath); + } + + // Rename temp to final + File.Move(tempFilePath, backupFilePath); + + Console.WriteLine($"Backup completed at {DateTime.Now}: " + + $"{data.StringStore.Count} string keys, " + + $"{data.ListStore.Count} list keys, " + + $"{data.HashStore.Count} hash keys"); + Console.WriteLine($"Saved to: {backupFileName}"); + + // Rotate backup files if needed + RotateBackupFiles(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error during backup file creation: {ex.Message}"); + } + } + else + { + Console.WriteLine("No data to back up"); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error during backup: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } + }); + } + #endregion + } +} \ No newline at end of file diff --git a/src/CommandLineParser.cs b/src/CommandLineParser.cs new file mode 100644 index 0000000..b27a858 --- /dev/null +++ b/src/CommandLineParser.cs @@ -0,0 +1,180 @@ +namespace Firefly +{ + public partial class Firefly + { + #region Command Line Parsing + static async Task ParseArguments(string[] args) + { + for (int i = 0; i < args.Length; i++) + { + string arg = args[i].ToLower(); + + switch (arg) + { + case "--port": + case "-p": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int port)) + { + serverPort = port; + Console.WriteLine($"Server port set to: {serverPort}"); + i++; // Skip the next arg (the port number) + } + else + { + Console.WriteLine("Warning: Invalid port specified, using default port 6379"); + } + break; + + case "--bind": + case "-b": + if (i + 1 < args.Length) + { + bindAddress = args[i + 1]; + if (bindAddress == "0.0.0.0") + { + Console.WriteLine("Server will listen on all network interfaces"); + } + else + { + Console.WriteLine($"Server bind address set to: {bindAddress}"); + } + i++; // Skip the next arg (the bind address) + } + else + { + Console.WriteLine("Warning: Bind address argument specified but no address provided"); + } + break; + + case "--password": + case "-pass": + if (i + 1 < args.Length) + { + serverPassword = args[i + 1]; + Console.WriteLine("Password authentication enabled"); + i++; // Skip the next arg (the password) + } + else + { + Console.WriteLine("Warning: Password argument specified but no password provided"); + } + break; + + case "--no-backup": + case "-nb": + backupsEnabled = false; + Console.WriteLine("Backups disabled"); + break; + + case "--backup-interval": + case "-bi": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int interval) && interval > 0) + { + autoBackupIntervalMinutes = interval; + Console.WriteLine($"Backup interval set to: {autoBackupIntervalMinutes} minutes"); + i++; // Skip the next arg (the interval) + } + else + { + Console.WriteLine($"Warning: Invalid backup interval specified, using default ({autoBackupIntervalMinutes} minutes)"); + } + break; + + case "--max-backups": + case "-mb": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int maxFiles) && maxFiles > 0) + { + maxBackupFiles = maxFiles; + Console.WriteLine($"Maximum backup files set to: {maxBackupFiles}"); + i++; // Skip the next arg (the count) + } + else + { + Console.WriteLine($"Warning: Invalid maximum backup files specified, using default ({maxBackupFiles})"); + } + break; + + case "--timeout": + case "-t": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int timeout) && timeout > 0) + { + connectionTimeoutSeconds = timeout; + Console.WriteLine($"Connection timeout set to: {connectionTimeoutSeconds} seconds"); + i++; // Skip the next arg (the timeout) + } + else + { + Console.WriteLine($"Warning: Invalid timeout specified, using default ({connectionTimeoutSeconds} seconds)"); + } + break; + + case "--pipeline-test": + case "-pt": + Console.WriteLine("Running in pipeline test mode..."); + + // Start the server in the background + var serverTask = Task.Run(async () => + { + await StartServerAsync(); + }); + + // Give the server a moment to start up + await Task.Delay(1000); + + // Run the pipeline test with all arguments + await PipelineTest.RunTest(args); + + // After the test completes, exit the application + Environment.Exit(0); + return; + + case "--help": + case "-h": + DisplayHelp(); + Environment.Exit(0); + break; + } + } + } + + static void DisplayHelp() + { + Console.WriteLine("\nFirefly Redis-compatible Server"); + Console.WriteLine("Usage: firefly [options]\n"); + Console.WriteLine("Options:"); + Console.WriteLine(" --port, -p Set server port (default: 6379)"); + Console.WriteLine(" --bind, -b
Set server bind address (default: 127.0.0.1)"); + Console.WriteLine(" --password, -pass Set server password for AUTH command"); + Console.WriteLine(" --no-backup, -nb Disable automatic backups"); + Console.WriteLine(" --backup-interval, -bi Set backup interval in minutes (default: 5)"); + Console.WriteLine(" --max-backups, -mb Set maximum number of backup files to keep (default: 10)"); + Console.WriteLine(" --timeout, -t Set client connection timeout in seconds (default: 300)"); + Console.WriteLine(" --pipeline-test, -pt Run in pipeline test mode"); + Console.WriteLine(" --help, -h Display this help message\n"); + + Console.WriteLine("Pipeline Test Options:"); + Console.WriteLine(" --host
Set test host (default: 127.0.0.1)"); + Console.WriteLine(" --clients Number of concurrent clients (default: 1)"); + Console.WriteLine(" --commands Number of commands per client (default: 10000)"); + Console.WriteLine(" --batch Batch size for batched test (default: 100)"); + Console.WriteLine(" --buffer Buffer size in KB (default: 1024)\n"); + + Console.WriteLine("Examples:"); + Console.WriteLine(" firefly --port 6380 Run server on port 6380"); + Console.WriteLine(" firefly --bind 0.0.0.0 Listen on all network interfaces"); + Console.WriteLine(" firefly --bind 192.168.1.10 Listen on a specific network interface"); + Console.WriteLine(" firefly --password secret123 Require password authentication"); + Console.WriteLine(" firefly -nb Run server with backups disabled"); + Console.WriteLine(" firefly -bi 10 Run server with backups every 10 minutes"); + Console.WriteLine(" firefly -mb 5 Keep only 5 most recent backup files"); + Console.WriteLine(" firefly -t 60 Set connection timeout to 60 seconds"); + Console.WriteLine(" firefly -pt Run pipeline performance test"); + Console.WriteLine(" firefly -pt --clients 4 Run pipeline test with 4 concurrent clients"); + Console.WriteLine(" firefly -pt --host 192.168.1.100 --port 6380 --password mypass"); + Console.WriteLine(" Run pipeline test with custom connection settings"); + Console.WriteLine(" firefly -pt --commands 5000 --batch 50"); + Console.WriteLine(" Run pipeline test with custom command count and batch size"); + } + #endregion + } +} \ No newline at end of file diff --git a/src/Firefly.cs b/src/Firefly.cs new file mode 100644 index 0000000..a1d2280 --- /dev/null +++ b/src/Firefly.cs @@ -0,0 +1,247 @@ +using System.Text; +using System.Collections.Concurrent; + +namespace Firefly +{ + #region Lock Class + // Simple lock class for synchronization + internal class Lock + { + private readonly System.Threading.Lock _lock = new(); + + public void Enter(Action action) + { + lock (_lock) + { + action(); + } + } + + public T Enter(Func action) + { + lock (_lock) + { + return action(); + } + } + } + #endregion + + partial class Firefly + { + #region Constants and Fields + // Sharding configuration - each shard will have its own ConcurrentDictionary + private const int ShardCount = 64; // Must be a power of 2 + private const int ShardMask = ShardCount - 1; // Used for fast modulo + + // In-memory storage for key-value pairs using ConcurrentDictionary for better performance + private static readonly ConcurrentDictionary[] stringStoreShards = CreateShards>(); + + // In-memory storage for lists using ConcurrentDictionary for better performance + private static readonly ConcurrentDictionary>[] listStoreShards = CreateShards>>(); + // Use ReaderWriterLockSlim for list operations to allow concurrent reads + private static readonly ReaderWriterLockSlim[] listStoreLocks = CreateListLocks(); + + // In-memory storage for hash tables using ConcurrentDictionary for better performance + private static readonly ConcurrentDictionary>[] hashStoreShards = CreateShards>>(); + // We no longer need locks for most hash operations + + // Global lock for backup operations + private static readonly Lock backupLock = new(); + + // Default settings that can be overridden by command-line arguments + private static int serverPort = 6379; // Default Redis port + private static string bindAddress = "127.0.0.1"; // Default to localhost only + private static bool backupsEnabled = true; + private static int autoBackupIntervalMinutes = 5; + private static int connectionTimeoutSeconds = 300; // 5 minutes default timeout + private static string serverPassword = ""; // Default: no password required + private static readonly Dictionary authenticatedClients = []; + private static readonly ConcurrentDictionary> clientCommandQueues = new(); + private static readonly int maxBatchSize = 1000; // Maximum number of commands to process in a batch + private static readonly int maxPipelineSize = 10000; // Maximum number of commands in a pipeline + private static readonly ConcurrentDictionary clientPipelineMode = new(); // Track if client is in pipeline mode + + // Network buffer settings + private static readonly int bufferSize = 20480; // Increased from 1024 for better performance + + // Backup settings + private static readonly string backupDirectory = "backups"; + private static readonly string backupFilePrefix = "firefly_data"; + private static readonly string backupFileExtension = ".fdb"; + private static int maxBackupFiles = 10; + private static System.Timers.Timer? backupTimer; + #endregion + + #region Main Entry Point + static async Task Main(string[] args) + { + // Parse command line arguments + await ParseArguments(args); + + // Initialize backup system if enabled + if (backupsEnabled) + { + Console.WriteLine("Initializing backup system..."); + InitializeBackupSystem(); + + // Load data from the latest backup + LoadDataFromBackup(); + } + + // Start the server + await StartServerAsync(); + } + #endregion + + #region Command Processing + static byte[] ProcessCommand(string message, string clientId) + { + // Trim whitespace and newlines + message = message.Trim(); + + // Split the command into parts, respecting quotes + string[] parts = SplitRespectingQuotes(message); + if (parts.Length == 0) + return Encoding.UTF8.GetBytes("-ERR no command specified\r\n"); + + string command = parts[0].ToUpperInvariant(); + + try + { + // Always allow AUTH command + if (command == "AUTH") + { + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'auth' command\r\n"); + } + return HandleAuthCommand(parts[1], clientId); + } + + // Always allow PING for connectivity checks + if (command == "PING") + { + return Encoding.UTF8.GetBytes("+PONG\r\n"); + } + + // Always allow QUIT command for clean disconnection + if (command == "QUIT") + { + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + // Handle PIPELINE command + if (command == "PIPELINE") + { + clientPipelineMode[clientId] = true; + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + // Handle EXEC command + if (command == "EXEC") + { + clientPipelineMode[clientId] = false; + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + // Check authentication for all other commands if password is set + if (!string.IsNullOrEmpty(serverPassword) && !IsAuthenticated(clientId)) + { + return Encoding.UTF8.GetBytes("-NOAUTH Authentication required.\r\n"); + } + + // Process different commands + switch (command) + { + case "ECHO": + if (parts.Length > 1) + { + string echoValue = parts[1]; + return Encoding.UTF8.GetBytes($"+{echoValue}\r\n"); + } + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'echo' command\r\n"); + + case "SET": + return HandleSetCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "GET": + return HandleGetCommand(parts.Length > 1 ? parts[1] : ""); + + case "DEL": + return HandleDelCommand(parts.Length > 1 ? parts[1] : ""); + + case "TYPE": + return HandleTypeCommand(parts.Length > 1 ? parts[1] : ""); + + case "LPUSH": + return HandleLPushCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "RPUSH": + return HandleRPushCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LPOP": + return HandleLPopCommand(parts.Length > 1 ? parts[1] : ""); + + case "RPOP": + return HandleRPopCommand(parts.Length > 1 ? parts[1] : ""); + + case "LINDEX": + return HandleLIndexCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LRANGE": + return HandleLRangeCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LSET": + return HandleLSetCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LPOS": + return HandleLPosCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LTRIM": + return HandleLTrimCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "LREM": + return HandleLRemCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "HSET": + return HandleHSetCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "HGET": + return HandleHGetCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "HDEL": + return HandleHDelCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "HEXISTS": + return HandleHExistsCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "HGETALL": + return HandleHGetAllCommand(parts.Length > 1 ? parts[1] : ""); + + case "HMSET": + return HandleHMSetCommand(parts.Length > 1 ? string.Join(" ", parts.Skip(1)) : ""); + + case "KEYS": + return HandleKeysCommand(parts.Length > 1 ? parts[1] : "*"); + + case "SAVE": + return HandleSaveCommand(); + + case "BGSAVE": + return HandleBgSaveCommand(); + + default: + return Encoding.UTF8.GetBytes($"-ERR unknown command '{command}'\r\n"); + } + } + catch (Exception ex) + { + // Catch any unexpected errors to prevent server crashes + Console.WriteLine($"Error processing command: {ex.Message}"); + return Encoding.UTF8.GetBytes($"-ERR internal server error: {ex.Message}\r\n"); + } + } + #endregion + } +} \ No newline at end of file diff --git a/src/FireflyData.cs b/src/FireflyData.cs new file mode 100644 index 0000000..b5c7269 --- /dev/null +++ b/src/FireflyData.cs @@ -0,0 +1,31 @@ +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Firefly +#pragma warning restore IDE0130 // Namespace does not match folder structure +{ + // Define FireflyData class to fix the missing type reference + /// + /// Container class for all Firefly database data used in serialization and backup operations. + /// + public class FireflyData + { + /// + /// Dictionary containing all string key-value pairs stored in the database. + /// + public Dictionary StringStore { get; set; } = []; + + /// + /// Dictionary containing all lists stored in the database. + /// + public Dictionary> ListStore { get; set; } = []; + + /// + /// Dictionary containing all hash tables stored in the database. + /// + public Dictionary> HashStore { get; set; } = []; + + /// + /// Timestamp when the backup was created. + /// + public DateTime BackupTime { get; init; } = DateTime.Now; + } +} \ No newline at end of file diff --git a/src/HashOperations.cs b/src/HashOperations.cs new file mode 100644 index 0000000..e53161c --- /dev/null +++ b/src/HashOperations.cs @@ -0,0 +1,262 @@ +using System.Text; +using System.Collections.Concurrent; + +namespace Firefly +{ + public partial class Firefly + { + #region Hash Operations + /// + /// Handles the HSET command which sets a field in a hash. + /// + /// Command arguments in format: "key field value" + /// 1 if the field was added, 0 if it was updated + static byte[] HandleHSetCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hset' command\r\n"); + } + + string key = parts[0]; + string field = parts[1]; + string value = parts.Length == 3 ? parts[2] : string.Join(" ", parts.Skip(2)); + + try + { + var hash = GetOrCreateHash(key); + bool isNewField = hash.TryAdd(field, value); + if (!isNewField) + { + hash[field] = value; + } + + return Encoding.UTF8.GetBytes($":{(isNewField ? 1 : 0)}\r\n"); + } + catch (InvalidOperationException ex) + { + // Handle the case where the key already exists with a different type + string? existingType = GetKeyType(key); + if (existingType != null) + { + return Encoding.UTF8.GetBytes($"-ERR key '{key}' already exists as type '{existingType}'\r\n"); + } + return Encoding.UTF8.GetBytes($"-ERR {ex.Message}\r\n"); + } + } + + /// + /// Handles the HGET command which retrieves the value of a field in a hash. + /// + /// Command arguments in format: "key field" + /// The value of the field, or nil if the field doesn't exist + static byte[] HandleHGetCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hget' command\r\n"); + } + + string key = parts[0]; + string field = parts[1]; + + if (HashStoreGet(key, out ConcurrentDictionary? hash) && + hash != null && hash.TryGetValue(field, out string? fieldValue)) + { + return Encoding.UTF8.GetBytes($"+{fieldValue}\r\n"); + } + + return Encoding.UTF8.GetBytes("$-1\r\n"); + } + + /// + /// Handles the HDEL command which removes a field from a hash. + /// + /// Command arguments in format: "key field" + /// 1 if the field was removed, 0 if it didn't exist + static byte[] HandleHDelCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hdel' command\r\n"); + } + + string key = parts[0]; + string field = parts[1]; + + if (HashStoreGet(key, out ConcurrentDictionary? hash) && hash != null) + { + bool removed = hash.TryRemove(field, out _); + + if (hash.IsEmpty) + { + HashStoreRemove(key); + } + + return Encoding.UTF8.GetBytes($":{(removed ? 1 : 0)}\r\n"); + } + + return Encoding.UTF8.GetBytes(":0\r\n"); + } + + /// + /// Handles the HEXISTS command which checks if a field exists in a hash. + /// + /// Command arguments in format: "key field" + /// 1 if the field exists, 0 if it doesn't + static byte[] HandleHExistsCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hexists' command\r\n"); + } + + string key = parts[0]; + string field = parts[1]; + + if (HashStoreGet(key, out ConcurrentDictionary? hash) && + hash != null && hash.TryGetValue(field, out _)) + { + return Encoding.UTF8.GetBytes(":1\r\n"); + } + + return Encoding.UTF8.GetBytes(":0\r\n"); + } + + /// + /// Handles the HGETALL command which retrieves all fields and values in a hash. + /// + /// Command arguments in format: "key" + /// All fields and values in the hash, or nil if the hash doesn't exist + static byte[] HandleHGetAllCommand(string args) + { + if (string.IsNullOrWhiteSpace(args)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hgetall' command\r\n"); + } + + string key = args.Trim(); + + if (HashStoreGet(key, out ConcurrentDictionary? hash) && + hash != null && !hash.IsEmpty) + { + var hashSnapshot = hash.ToArray(); + StringBuilder response = new(); + response.Append("*\r\n"); + + foreach (var kvp in hashSnapshot) + { + response.Append($"+{kvp.Key}\r\n"); + response.Append($"+{kvp.Value}\r\n"); + } + + return Encoding.UTF8.GetBytes(response.ToString()); + } + + return Encoding.UTF8.GetBytes("*\r\n"); + } + + /// + /// Handles the HMSET command which sets multiple fields in a hash. + /// + /// Command arguments in format: "key field value [field value ...]" + /// OK on success, error if arguments are invalid + static byte[] HandleHMSetCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3 || (parts.Length - 1) % 2 != 0) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'hmset' command\r\n"); + } + + string key = parts[0]; + var hash = GetOrCreateHash(key); + + try + { + for (int i = 1; i < parts.Length; i += 2) + { + string field = parts[i]; + if (i + 1 < parts.Length) + { + string value = parts[i + 1]; + if (value.StartsWith('"') && value.EndsWith('"') && value.Length >= 2) + { + value = value[1..^1]; + } + hash[field] = value; + } + } + } + catch (Exception ex) + { + return Encoding.UTF8.GetBytes($"-ERR internal error: {ex.Message}\r\n"); + } + + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + #region Hash Store Helpers + /// + /// Gets or creates a hash for a given key. + /// + /// The key to get or create the hash for + /// The hash + private static ConcurrentDictionary GetOrCreateHash(string key) + { + int shardIndex = GetShardIndex(key); + + // If the key doesn't exist in this shard, check if it exists in any other store + if (!hashStoreShards[shardIndex].ContainsKey(key)) + { + // Check if key exists in any other store + if (!EnsureKeyDoesNotExist(key, "hash")) + { + throw new InvalidOperationException($"Key '{key}' already exists with a different type"); + } + } + + return hashStoreShards[shardIndex].GetOrAdd(key, _ => new ConcurrentDictionary()); + } + + /// + /// Checks if a hash exists for a given key. + /// + /// The key to check the hash for + /// True if the hash exists, false otherwise + private static bool HashStoreExists(string key) + { + int shardIndex = GetShardIndex(key); + return hashStoreShards[shardIndex].ContainsKey(key); + } + + /// + /// Gets a hash for a given key. + /// + /// The key to get the hash for + /// The hash + /// True if the hash was found, false otherwise + private static bool HashStoreGet(string key, out ConcurrentDictionary? value) + { + int shardIndex = GetShardIndex(key); + return hashStoreShards[shardIndex].TryGetValue(key, out value); + } + + /// + /// Removes a hash for a given key. + /// + /// The key to remove the hash for + /// True if the hash was removed, false otherwise + private static bool HashStoreRemove(string key) + { + int shardIndex = GetShardIndex(key); + return hashStoreShards[shardIndex].TryRemove(key, out _); + } + #endregion + #endregion + } +} \ No newline at end of file diff --git a/src/Helpers.cs b/src/Helpers.cs new file mode 100644 index 0000000..08357ea --- /dev/null +++ b/src/Helpers.cs @@ -0,0 +1,308 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace Firefly +{ + public partial class Firefly + { + #region Utility Methods + /// + /// Splits a string respecting quoted sections. + /// + /// The input string to split + /// An array of tokens from the input string + static string[] SplitRespectingQuotes(string input) + { + var result = new List(); + bool inQuotes = false; + StringBuilder currentToken = new(); + + for (int i = 0; i < input.Length; i++) + { + char c = input[i]; + + if (c == '"') + { + // Toggle quote state + inQuotes = !inQuotes; + } + else if (c == ' ' && !inQuotes) + { + // Space outside quotes - token boundary + if (currentToken.Length > 0) + { + // Add the completed token + string token = currentToken.ToString(); + result.Add(token); + currentToken.Clear(); + } + // Skip the space + continue; + } + + // Add the character to the current token + currentToken.Append(c); + } + + // Add final token if any + if (currentToken.Length > 0) + { + result.Add(currentToken.ToString()); + } + + return [.. result]; + } + + [GeneratedRegex(@"([^\s""]+)|""([^""]*)""")] + private static partial Regex MyRegex(); + #endregion + + #region Sharding and Storage Helpers + /// + /// Creates an array of shards of type T. + /// + /// The type of the shards + /// An array of shards + private static T[] CreateShards() where T : new() + { + var shards = new T[ShardCount]; + for (int i = 0; i < ShardCount; i++) + { + shards[i] = new T(); + } + return shards; + } + + /// + /// Creates an array of ReaderWriterLockSlim instances for list operations. + /// + /// An array of locks + private static ReaderWriterLockSlim[] CreateListLocks() + { + var locks = new ReaderWriterLockSlim[ShardCount]; + for (int i = 0; i < ShardCount; i++) + { + locks[i] = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion); + } + return locks; + } + + /// + /// Gets the shard index for a given key. + /// + /// The key to get the shard index for + /// The shard index + private static int GetShardIndex(string key) + { + return key.GetHashCode() & 0x7FFFFFFF & ShardMask; // Ensure positive hash and fast modulo + } + #endregion + + #region Lock Helpers + /// + /// Creates an array of locks. + /// + /// An array of locks + private static object[] CreateLocks() + { + var locks = new object[ShardCount]; + for (int i = 0; i < ShardCount; i++) + { + locks[i] = new object(); + } + return locks; + } + #endregion + + #region Key Management Helpers + /// + /// Checks if a key exists in any store (string, list, or hash). + /// + /// The key to check + /// True if the key exists in any store, false otherwise + private static bool KeyExistsInAnyStore(string key) + { + // Check string store + if (StringStoreExists(key)) + { + return true; + } + + // Check list store + if (ListStoreExists(key)) + { + return true; + } + + // Check hash store + if (HashStoreExists(key)) + { + return true; + } + + return false; + } + + /// + /// Gets the type of a key if it exists in any store. + /// + /// The key to check + /// The type of the key ("string", "list", "hash", or null if not found) + private static string? GetKeyType(string key) + { + // Check string store + if (StringStoreExists(key)) + { + return "string"; + } + + // Check list store + if (ListStoreExists(key)) + { + return "list"; + } + + // Check hash store + if (HashStoreExists(key)) + { + return "hash"; + } + + return null; + } + + /// + /// Ensures a key doesn't exist in any store before creating it in the target store. + /// + /// The key to check + /// The type of store where the key will be created + /// True if the key can be created, false if it already exists with a different type + private static bool EnsureKeyDoesNotExist(string key, string targetType) + { + string? existingType = GetKeyType(key); + if (existingType != null) + { + return false; // Key already exists with a different type + } + return true; // Key doesn't exist in any store + } + + /// + /// Handles the DEL command which removes a key from all stores (string, list, hash). + /// + /// The key to delete + /// The number of keys that were removed + public static byte[] HandleDelCommand(string key) + { + if (string.IsNullOrEmpty(key)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'del' command\r\n"); + } + + int deletedCount = 0; + + // Try to delete from string store + if (StringStoreRemove(key)) + { + deletedCount++; + } + + // Try to delete from list store + if (ListStoreRemove(key)) + { + deletedCount++; + } + + // Try to delete from hash store + if (HashStoreRemove(key)) + { + deletedCount++; + } + + return Encoding.UTF8.GetBytes($":{deletedCount}\r\n"); + } + + /// + /// Handles the TYPE command which returns the type of a key. + /// + /// The key to check the type of + /// The type of the key as a string response + public static byte[] HandleTypeCommand(string key) + { + if (string.IsNullOrEmpty(key)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'type' command\r\n"); + } + + // Check string store + if (StringStoreExists(key)) + { + return Encoding.UTF8.GetBytes("+string\r\n"); + } + + // Check list store + if (ListStoreExists(key)) + { + return Encoding.UTF8.GetBytes("+list\r\n"); + } + + // Check hash store + if (HashStoreExists(key)) + { + return Encoding.UTF8.GetBytes("+hash\r\n"); + } + + // Key doesn't exist + return Encoding.UTF8.GetBytes("+none\r\n"); + } + + /// + /// Handles the KEYS command which returns all keys matching a pattern. + /// + /// The pattern to match + /// A list of keys matching the pattern + public static byte[] HandleKeysCommand(string pattern) + { + try + { + var matchingKeys = new List(); + + // Convert pattern to regex + var regex = new Regex( + "^" + Regex.Escape(pattern).Replace("\\*", ".*").Replace("\\?", ".") + "$", + RegexOptions.Compiled + ); + + // Search in string store + foreach (var shard in stringStoreShards) + { + matchingKeys.AddRange(shard.Keys.Where(k => regex.IsMatch(k))); + } + + // Search in list store + foreach (var shard in listStoreShards) + { + matchingKeys.AddRange(shard.Keys.Where(k => regex.IsMatch(k))); + } + + // Search in hash store + foreach (var shard in hashStoreShards) + { + matchingKeys.AddRange(shard.Keys.Where(k => regex.IsMatch(k))); + } + + // Sort keys for consistent results + matchingKeys.Sort(); + + // Return keys as newline-separated string + string result = string.Join("\n", matchingKeys); + return Encoding.UTF8.GetBytes($"+{result}\r\n"); + } + catch (Exception ex) + { + return Encoding.UTF8.GetBytes($"-ERR error executing KEYS command: {ex.Message}\r\n"); + } + } + #endregion + } +} \ No newline at end of file diff --git a/src/ListOperations.cs b/src/ListOperations.cs new file mode 100644 index 0000000..716af73 --- /dev/null +++ b/src/ListOperations.cs @@ -0,0 +1,624 @@ +using System.Text; + +namespace Firefly +{ + public partial class Firefly + { + #region List Operations + /// + /// Handles the LPUSH command which adds an element to the left of a list. + /// + /// Command arguments in format: "key value" + /// The length of the list after the push operation + static byte[] HandleLPushCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lpush' command\r\n"); + } + + string key = parts[0]; + string value = parts[1]; + + try + { + // Use helper function to safely modify the list with write lock + int newLength = 0; + ListStoreWithWriteLock(key, list => + { + list.Insert(0, value); + newLength = list.Count; + }); + + return Encoding.UTF8.GetBytes($":{newLength}\r\n"); + } + catch (InvalidOperationException ex) + { + // Handle the case where the key already exists with a different type + string? existingType = GetKeyType(key); + if (existingType != null) + { + return Encoding.UTF8.GetBytes($"-ERR key '{key}' already exists as type '{existingType}'\r\n"); + } + return Encoding.UTF8.GetBytes($"-ERR {ex.Message}\r\n"); + } + } + + /// + /// Handles the RPUSH command which adds values to the tail of a list. + /// + /// Command arguments in format: "key value1 [value2 ...]" + /// Response indicating the new length of the list + static byte[] HandleRPushCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'rpush' command\r\n"); + } + + string key = parts[0]; + List result = []; + + // Use helper function to safely modify the list with write lock + ListStoreWithWriteLock(key, list => + { + // Add all values to the end of the list + for (int i = 1; i < parts.Length; i++) + { + list.Add(parts[i]); + } + result.Add(list.Count.ToString()); + }); + + return Encoding.UTF8.GetBytes($":{result[0]}\r\n"); + } + + /// + /// Handles the LPOP command which removes and returns the first element of a list. + /// + /// Command arguments containing the key + /// The popped value or nil if the list is empty + static byte[] HandleLPopCommand(string args) + { + if (string.IsNullOrWhiteSpace(args)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lpop' command\r\n"); + } + + string key = args.Trim(); + List result = []; + + // Use helper function to safely modify the list with write lock + ListStoreWithWriteLock(key, list => + { + if (list.Count > 0) + { + // Remove and store the first element + string value = list[0]; + list.RemoveAt(0); + result.Add(value); + + // Clean up empty lists + if (list.Count == 0) + { + ListStoreRemove(key); + } + } + }); + + return result.Count > 0 + ? Encoding.UTF8.GetBytes($"+{result[0]}\r\n") + : Encoding.UTF8.GetBytes("$-1\r\n"); + } + + /// + /// Handles the RPOP command which removes and returns the last element of a list. + /// + /// Command arguments containing the key + /// The popped value or nil if the list is empty + static byte[] HandleRPopCommand(string args) + { + if (string.IsNullOrWhiteSpace(args)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'rpop' command\r\n"); + } + + string key = args.Trim(); + List result = []; + + // Use helper function to safely modify the list with write lock + ListStoreWithWriteLock(key, list => + { + if (list.Count > 0) + { + // Remove and store the last element + int lastIndex = list.Count - 1; + string value = list[lastIndex]; + list.RemoveAt(lastIndex); + result.Add(value); + + // Clean up empty lists + if (list.Count == 0) + { + ListStoreRemove(key); + } + } + }); + + return result.Count > 0 + ? Encoding.UTF8.GetBytes($"+{result[0]}\r\n") + : Encoding.UTF8.GetBytes("$-1\r\n"); + } + + /// + /// Handles the LRANGE command which returns a range of elements from a list. + /// + /// Command arguments in format: "key start stop" + /// Array of elements in the specified range + static byte[] HandleLRangeCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lrange' command\r\n"); + } + + string key = parts[0]; + if (!int.TryParse(parts[1], out int start) || !int.TryParse(parts[2], out int stop)) + { + return Encoding.UTF8.GetBytes("-ERR value is not an integer or out of range\r\n"); + } + + // Use helper function to safely read the list with read lock + return ListStoreWithReadLock(key, list => + { + if (list.Count == 0) + { + return Encoding.UTF8.GetBytes("*\r\n"); + } + + // Handle negative indices (counting from the end) + if (start < 0) start = list.Count + start; + if (stop < 0) stop = list.Count + stop; + + // Ensure indices are within bounds + start = Math.Max(start, 0); + stop = Math.Min(stop, list.Count - 1); + + if (start > stop) + { + return Encoding.UTF8.GetBytes("*\r\n"); + } + + // Build response with all elements in range + StringBuilder response = new(); + response.Append("*\r\n"); + + for (int i = start; i <= stop; i++) + { + response.Append($"+{list[i]}\r\n"); + } + + return Encoding.UTF8.GetBytes(response.ToString()); + }) ?? Encoding.UTF8.GetBytes("*\r\n"); + } + + /// + /// Handles the LINDEX command which returns an element from a list by its index. + /// + /// Command arguments in format: "key index" + /// The element at the specified index or nil if not found + static byte[] HandleLIndexCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lindex' command\r\n"); + } + + string key = parts[0]; + if (!int.TryParse(parts[1], out int index)) + { + return Encoding.UTF8.GetBytes("-ERR value is not an integer or out of range\r\n"); + } + + // Use helper function to safely read the list with read lock + return ListStoreWithReadLock(key, list => + { + if (list.Count == 0) + { + return Encoding.UTF8.GetBytes("$-1\r\n"); + } + + // Handle negative index (counting from the end) + if (index < 0) + { + index = list.Count + index; + } + + // Check if index is within bounds + if (index < 0 || index >= list.Count) + { + return Encoding.UTF8.GetBytes("$-1\r\n"); + } + + return Encoding.UTF8.GetBytes($"+{list[index]}\r\n"); + }) ?? Encoding.UTF8.GetBytes("$-1\r\n"); + } + + /// + /// Handles the LSET command which sets the value of an element in a list by its index. + /// + /// Command arguments in format: "key index value" + /// OK on success, error if index is out of range + static byte[] HandleLSetCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lset' command\r\n"); + } + + string key = parts[0]; + if (!int.TryParse(parts[1], out int index)) + { + return Encoding.UTF8.GetBytes("-ERR value is not an integer or out of range\r\n"); + } + string value = parts[2]; + + bool success = false; + // Use helper function to safely modify the list with write lock + ListStoreWithWriteLock(key, list => + { + if (list.Count == 0) + { + return; + } + + // Handle negative index (counting from the end) + if (index < 0) + { + index = list.Count + index; + } + + // Check if index is within bounds + if (index < 0 || index >= list.Count) + { + return; + } + + list[index] = value; + success = true; + }); + + return success + ? Encoding.UTF8.GetBytes("+OK\r\n") + : Encoding.UTF8.GetBytes("-ERR index out of range\r\n"); + } + + /// + /// Handles the LPOS command which returns the position of an element in a list. + /// + /// Command arguments in format: "key element [RANK rank] [MAXLEN len]" + /// The position of the element or nil if not found + static byte[] HandleLPosCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lpos' command\r\n"); + } + + string key = parts[0]; + string element = parts[1]; + + // Optional parameters + int rank = 1; // Default to first occurrence + int maxlen = 0; // 0 means no limit + + // Parse optional parameters + for (int i = 2; i < parts.Length; i++) + { + if (parts[i].Equals("RANK", StringComparison.OrdinalIgnoreCase) && i + 1 < parts.Length) + { + if (int.TryParse(parts[++i], out int parsedRank)) + { + rank = parsedRank; + } + } + else if (parts[i].Equals("MAXLEN", StringComparison.OrdinalIgnoreCase) && i + 1 < parts.Length) + { + if (int.TryParse(parts[++i], out int parsedMaxlen)) + { + maxlen = parsedMaxlen; + } + } + } + + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterReadLock(); + try + { + if (!listStoreShards[shardIndex].TryGetValue(key, out List? list) || list == null) + { + return Encoding.UTF8.GetBytes("$-1\r\n"); // Key doesn't exist + } + + int found = 0; + for (int i = 0; i < list.Count; i++) + { + if (maxlen > 0 && i >= maxlen) + { + break; + } + + if (list[i] == element) + { + found++; + if (found == Math.Abs(rank)) + { + return Encoding.UTF8.GetBytes($":{i}\r\n"); + } + } + } + + return Encoding.UTF8.GetBytes("$-1\r\n"); // Element not found + } + finally + { + listStoreLocks[shardIndex].ExitReadLock(); + } + } + + + /// + /// Handles the LTRIM command which trims a list to the specified range. + /// + /// Command arguments in format: "key start stop" + /// OK on success, error if arguments are invalid + static byte[] HandleLTrimCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'ltrim' command\r\n"); + } + + string key = parts[0]; + if (!int.TryParse(parts[1], out int start) || !int.TryParse(parts[2], out int stop)) + { + return Encoding.UTF8.GetBytes("-ERR value is not an integer or out of range\r\n"); + } + + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterWriteLock(); + try + { + if (!listStoreShards[shardIndex].TryGetValue(key, out List? list) || list == null) + { + return Encoding.UTF8.GetBytes("+OK\r\n"); // Non-existent key is treated as empty list + } + + // Handle negative indices + if (start < 0) start = list.Count + start; + if (stop < 0) stop = list.Count + stop; + + // Normalize boundaries + start = Math.Max(start, 0); + stop = Math.Min(stop, list.Count - 1); + + if (start > stop || start >= list.Count) + { + // Clear the list if range is invalid + list.Clear(); + listStoreShards[shardIndex].TryRemove(key, out _); + } + else + { + // Calculate the new range + int newLength = stop - start + 1; + if (start > 0 || stop < list.Count - 1) + { + var trimmed = list.GetRange(start, newLength); + list.Clear(); + list.AddRange(trimmed); + } + } + + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + finally + { + listStoreLocks[shardIndex].ExitWriteLock(); + } + } + + /// + /// Handles the LREM command which removes elements equal to the given value from a list. + /// + /// Command arguments in format: "key count element" + /// The number of removed elements + static byte[] HandleLRemCommand(string args) + { + string[] parts = SplitRespectingQuotes(args); + if (parts.Length < 3) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'lrem' command\r\n"); + } + + string key = parts[0]; + if (!int.TryParse(parts[1], out int count)) + { + return Encoding.UTF8.GetBytes("-ERR value is not an integer or out of range\r\n"); + } + string element = parts[2]; + + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterWriteLock(); + try + { + if (!listStoreShards[shardIndex].TryGetValue(key, out List? list) || list == null) + { + return Encoding.UTF8.GetBytes(":0\r\n"); // Key doesn't exist + } + + int removed = 0; + if (count > 0) + { + // Remove count occurrences from head to tail + for (int i = 0; i < list.Count && removed < count; i++) + { + if (list[i] == element) + { + list.RemoveAt(i); + removed++; + i--; // Adjust index after removal + } + } + } + else if (count < 0) + { + // Remove |count| occurrences from tail to head + for (int i = list.Count - 1; i >= 0 && removed < -count; i--) + { + if (list[i] == element) + { + list.RemoveAt(i); + removed++; + } + } + } + else // count == 0 + { + // Remove all occurrences + removed = list.RemoveAll(x => x == element); + } + + // Remove the key if the list is empty + if (list.Count == 0) + { + listStoreShards[shardIndex].TryRemove(key, out _); + } + + return Encoding.UTF8.GetBytes($":{removed}\r\n"); + } + finally + { + listStoreLocks[shardIndex].ExitWriteLock(); + } + } + + #region List Store Helpers + /// + /// Gets or creates a list for a given key. + /// + /// The key to get or create the list for + /// The list + private static List GetOrCreateList(string key) + { + int shardIndex = GetShardIndex(key); + + // If the key doesn't exist in this shard, check if it exists in any other store + if (!listStoreShards[shardIndex].ContainsKey(key)) + { + // Check if key exists in any other store + if (!EnsureKeyDoesNotExist(key, "list")) + { + throw new InvalidOperationException($"Key '{key}' already exists with a different type"); + } + } + + return listStoreShards[shardIndex].GetOrAdd(key, _ => []); + } + + /// + /// Checks if a list exists for a given key. + /// + /// The key to check the list for + /// True if the list exists, false otherwise + private static bool ListStoreExists(string key) + { + int shardIndex = GetShardIndex(key); + return listStoreShards[shardIndex].ContainsKey(key); + } + + /// + /// Gets a list for a given key. + /// + /// The key to get the list for + /// The list + /// True if the list was found, false otherwise + private static bool ListStoreGet(string key, out List? value) + { + int shardIndex = GetShardIndex(key); + return listStoreShards[shardIndex].TryGetValue(key, out value); + } + + /// + /// Removes a list for a given key. + /// + /// The key to remove the list for + /// True if the list was removed, false otherwise + private static bool ListStoreRemove(string key) + { + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterWriteLock(); + try + { + return listStoreShards[shardIndex].TryRemove(key, out _); + } + finally + { + listStoreLocks[shardIndex].ExitWriteLock(); + } + } + + /// + /// Executes an action with a write lock on a list for a given key. + /// + /// The key to execute the action on + /// The action to execute + private static void ListStoreWithWriteLock(string key, Action> action) + { + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterWriteLock(); + try + { + var list = GetOrCreateList(key); + action(list); + } + finally + { + listStoreLocks[shardIndex].ExitWriteLock(); + } + } + + /// + /// Executes an action with a read lock on a list for a given key. + /// + /// The type of the result + /// The key to execute the action on + /// The action to execute + private static T ListStoreWithReadLock(string key, Func, T> action) + { + int shardIndex = GetShardIndex(key); + listStoreLocks[shardIndex].EnterReadLock(); + try + { + if (ListStoreGet(key, out List? list) && list != null) + { + return action(list); + } + return default!; + } + finally + { + listStoreLocks[shardIndex].ExitReadLock(); + } + } + #endregion + #endregion + } +} \ No newline at end of file diff --git a/src/PipelineTest.cs b/src/PipelineTest.cs new file mode 100644 index 0000000..f1d8b54 --- /dev/null +++ b/src/PipelineTest.cs @@ -0,0 +1,407 @@ +using System.Diagnostics; +using System.Net.Sockets; +using System.Text; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Firefly +#pragma warning restore IDE0130 // Namespace does not match folder structure +{ + /// + /// Test client for verifying pipelining and batching functionality + /// + public class PipelineTest + { + private static string Host = "127.0.0.1"; + private static int Port = 6379; + private static string Password = ""; + private static int CommandCount = 10000; + private static int BatchSize = 100; + private static int BufferSize = 1024 * 1024; // 1MB buffer + private static int ClientCount = 1; + + /// + /// Runs the pipeline test + /// + public static async Task RunTest(string[] args) + { + // Parse command line arguments + ParseArguments(args); + + Console.WriteLine("\nStarting Pipeline Test..."); + Console.WriteLine($"Host: {Host}"); + Console.WriteLine($"Port: {Port}"); + Console.WriteLine($"Clients: {ClientCount}"); + if (!string.IsNullOrEmpty(Password)) + { + Console.WriteLine("Authentication: Enabled"); + } + Console.WriteLine($"Command Count: {CommandCount}"); + Console.WriteLine($"Batch Size: {BatchSize}"); + Console.WriteLine($"Buffer Size: {BufferSize / 1024}KB\n"); + + // Test 1: Sequential commands (no pipelining) + await RunSequentialTest(); + + // Test 2: Pipelined commands + await RunPipelinedTest(); + + // Test 3: Batched commands + await RunBatchedTest(); + + Console.WriteLine("\nPipeline Test completed."); + } + + private static void ParseArguments(string[] args) + { + for (int i = 0; i < args.Length; i++) + { + string arg = args[i].ToLower(); + + switch (arg) + { + case "--host": + case "-h": + if (i + 1 < args.Length) + { + Host = args[i + 1]; + i++; + } + break; + + case "--port": + case "-p": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int port)) + { + Port = port; + i++; + } + break; + + case "--password": + case "-pass": + if (i + 1 < args.Length) + { + Password = args[i + 1]; + i++; + } + break; + + case "--commands": + case "-c": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int commands)) + { + CommandCount = commands; + i++; + } + break; + + case "--batch": + case "-b": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int batch)) + { + BatchSize = batch; + i++; + } + break; + + case "--buffer": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int buffer)) + { + BufferSize = buffer * 1024; // Convert KB to bytes + i++; + } + break; + + case "--clients": + if (i + 1 < args.Length && int.TryParse(args[i + 1], out int clients)) + { + ClientCount = clients; + i++; + } + break; + } + } + } + + private static async Task Authenticate(NetworkStream stream) + { + if (!string.IsNullOrEmpty(Password)) + { + string authCommand = $"AUTH {Password}\r\n"; + byte[] authData = Encoding.UTF8.GetBytes(authCommand); + await stream.WriteAsync(authData); + + var buffer = new byte[BufferSize]; + int bytesRead = await stream.ReadAsync(buffer); + string response = Encoding.UTF8.GetString(buffer, 0, bytesRead); + + if (!response.StartsWith("+OK")) + { + throw new Exception($"Authentication failed: {response}"); + } + } + } + + /// + /// Read responses from the stream until we have the expected count + /// + private static async Task ReadResponses(NetworkStream stream, byte[] buffer, int expectedCount, string testType) + { + var responseBuilder = new StringBuilder(); + int totalResponses = 0; + int retryCount = 0; + const int maxRetries = 10; + + while (totalResponses < expectedCount && retryCount < maxRetries) + { + if (stream.DataAvailable) + { + int bytesRead = await stream.ReadAsync(buffer); + if (bytesRead > 0) + { + string response = Encoding.UTF8.GetString(buffer, 0, bytesRead); + responseBuilder.Append(response); + + // Count complete responses + int newResponses = response.Split(["\r\n"], StringSplitOptions.RemoveEmptyEntries).Length; + totalResponses += newResponses; + + if (totalResponses % 50 == 0) + { + Console.WriteLine($"{testType}: Received {totalResponses}/{expectedCount} responses"); + } + + retryCount = 0; // Reset retry count on successful read + } + } + else + { + await Task.Delay(100); // Wait a bit for more data + retryCount++; + } + } + + if (totalResponses < expectedCount) + { + Console.WriteLine($"Warning: Only received {totalResponses}/{expectedCount} responses"); + } + + return totalResponses; + } + + /// + /// Test with sequential commands (no pipelining) + /// + private static async Task RunSequentialTest() + { + Console.WriteLine("=== Sequential Test ==="); + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var tasks = new List(); + var clientResults = new List<(int clientId, int responses, double opsPerSecond)>(); + + for (int clientId = 0; clientId < ClientCount; clientId++) + { + tasks.Add(Task.Run(async () => + { + using var client = new TcpClient(); + await client.ConnectAsync(Host, Port); + using var stream = client.GetStream(); + var buffer = new byte[BufferSize]; + int totalResponses = 0; + + // Authenticate if password is set + await Authenticate(stream); + + for (int i = 0; i < CommandCount; i++) + { + string key = $"seq:client{clientId}:key:{i}"; + string value = $"value:{i}"; + + // SET command + string setCommand = $"SET {key} {value}\r\n"; + byte[] setData = Encoding.UTF8.GetBytes(setCommand); + await stream.WriteAsync(setData); + totalResponses += await ReadResponses(stream, buffer, 1, $"Sequential-Client{clientId}"); + + // GET command + string getCommand = $"GET {key}\r\n"; + byte[] getData = Encoding.UTF8.GetBytes(getCommand); + await stream.WriteAsync(getData); + totalResponses += await ReadResponses(stream, buffer, 1, $"Sequential-Client{clientId}"); + + if (i % 10 == 0) + { + Console.WriteLine($"Sequential-Client{clientId}: Processed {i}/{CommandCount} commands"); + } + } + + clientResults.Add((clientId, totalResponses, (CommandCount * 2.0) / (stopwatch.ElapsedMilliseconds / 1000.0))); + })); + } + + await Task.WhenAll(tasks); + stopwatch.Stop(); + + // Print individual client results + foreach (var (clientId, responses, opsPerSecond) in clientResults) + { + Console.WriteLine($"Sequential-Client{clientId}:"); + Console.WriteLine($" Total responses: {responses}"); + Console.WriteLine($" Operations per second: {opsPerSecond:F2}"); + } + + // Print aggregate results + double totalOpsPerSecond = clientResults.Sum(r => r.opsPerSecond); + Console.WriteLine($"\nSequential Test Summary:"); + Console.WriteLine($"Total operations per second: {totalOpsPerSecond:F2}"); + Console.WriteLine($"Average operations per second per client: {totalOpsPerSecond / ClientCount:F2}\n"); + } + + /// + /// Test with pipelined commands + /// + private static async Task RunPipelinedTest() + { + Console.WriteLine("=== Pipelined Test ==="); + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var tasks = new List(); + var clientResults = new List<(int clientId, int responses, double opsPerSecond)>(); + + for (int clientId = 0; clientId < ClientCount; clientId++) + { + tasks.Add(Task.Run(async () => + { + using var client = new TcpClient(); + await client.ConnectAsync(Host, Port); + using var stream = client.GetStream(); + var buffer = new byte[BufferSize]; + + // Authenticate if password is set + await Authenticate(stream); + + // Enter pipeline mode + string pipelineCommand = "PIPELINE\r\n"; + byte[] pipelineData = Encoding.UTF8.GetBytes(pipelineCommand); + await stream.WriteAsync(pipelineData); + await ReadResponses(stream, buffer, 1, $"Pipeline-Client{clientId}"); + + Console.WriteLine($"Pipeline-Client{clientId}: Sending commands..."); + + // Send all commands + var allCommands = new StringBuilder(); + for (int i = 0; i < CommandCount; i++) + { + string key = $"pipe:client{clientId}:key:{i}"; + string value = $"value:{i}"; + allCommands.Append($"SET {key} {value}\r\n"); + allCommands.Append($"GET {key}\r\n"); + } + + byte[] commandsData = Encoding.UTF8.GetBytes(allCommands.ToString()); + await stream.WriteAsync(commandsData); + + // Execute pipeline + string execCommand = "EXEC\r\n"; + byte[] execData = Encoding.UTF8.GetBytes(execCommand); + await stream.WriteAsync(execData); + + // Read all responses + int totalResponses = await ReadResponses(stream, buffer, CommandCount * 2 + 1, $"Pipeline-Client{clientId}"); + clientResults.Add((clientId, totalResponses, (CommandCount * 2.0) / (stopwatch.ElapsedMilliseconds / 1000.0))); + })); + } + + await Task.WhenAll(tasks); + stopwatch.Stop(); + + // Print individual client results + foreach (var (clientId, responses, opsPerSecond) in clientResults) + { + Console.WriteLine($"Pipeline-Client{clientId}:"); + Console.WriteLine($" Total responses: {responses}"); + Console.WriteLine($" Operations per second: {opsPerSecond:F2}"); + } + + // Print aggregate results + double totalOpsPerSecond = clientResults.Sum(r => r.opsPerSecond); + Console.WriteLine($"\nPipelined Test Summary:"); + Console.WriteLine($"Total operations per second: {totalOpsPerSecond:F2}"); + Console.WriteLine($"Average operations per second per client: {totalOpsPerSecond / ClientCount:F2}\n"); + } + + /// + /// Test with batched commands + /// + private static async Task RunBatchedTest() + { + Console.WriteLine("=== Batched Test ==="); + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var tasks = new List(); + var clientResults = new List<(int clientId, int responses, double opsPerSecond)>(); + + for (int clientId = 0; clientId < ClientCount; clientId++) + { + tasks.Add(Task.Run(async () => + { + using var client = new TcpClient(); + await client.ConnectAsync(Host, Port); + using var stream = client.GetStream(); + var buffer = new byte[BufferSize]; + int totalResponses = 0; + + // Authenticate if password is set + await Authenticate(stream); + + for (int batch = 0; batch < CommandCount; batch += BatchSize) + { + int currentBatchSize = Math.Min(BatchSize, CommandCount - batch); + var batchCommands = new StringBuilder(); + + for (int i = 0; i < currentBatchSize; i++) + { + int index = batch + i; + string key = $"batch:client{clientId}:key:{index}"; + string value = $"value:{index}"; + batchCommands.Append($"SET {key} {value}\r\n"); + batchCommands.Append($"GET {key}\r\n"); + } + + byte[] batchData = Encoding.UTF8.GetBytes(batchCommands.ToString()); + await stream.WriteAsync(batchData); + + int responses = await ReadResponses(stream, buffer, currentBatchSize * 2, $"Batch-Client{clientId}"); + totalResponses += responses; + + Console.WriteLine($"Batch-Client{clientId}: Processed batch {batch / BatchSize + 1}/{Math.Ceiling((double)CommandCount / BatchSize)} " + + $"({responses} responses)"); + } + + clientResults.Add((clientId, totalResponses, (CommandCount * 2.0) / (stopwatch.ElapsedMilliseconds / 1000.0))); + })); + } + + await Task.WhenAll(tasks); + stopwatch.Stop(); + + // Print individual client results + foreach (var (clientId, responses, opsPerSecond) in clientResults) + { + Console.WriteLine($"Batch-Client{clientId}:"); + Console.WriteLine($" Total responses: {responses}"); + Console.WriteLine($" Operations per second: {opsPerSecond:F2}"); + } + + // Print aggregate results + double totalOpsPerSecond = clientResults.Sum(r => r.opsPerSecond); + Console.WriteLine($"\nBatched Test Summary:"); + Console.WriteLine($"Total operations per second: {totalOpsPerSecond:F2}"); + Console.WriteLine($"Average operations per second per client: {totalOpsPerSecond / ClientCount:F2}\n"); + } + } +} \ No newline at end of file diff --git a/src/Protocol/FireflyProtocol.cs b/src/Protocol/FireflyProtocol.cs new file mode 100644 index 0000000..a6738e5 --- /dev/null +++ b/src/Protocol/FireflyProtocol.cs @@ -0,0 +1,211 @@ +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Firefly.Protocol +#pragma warning restore IDE0130 // Namespace does not match folder structure +{ + /// + /// Defines the Firefly wire protocol specification for 3rd party clients + /// + public static class FireflyProtocol + { + /// + /// The current version of the Firefly protocol + /// + public const string PROTOCOL_VERSION = "1.0"; + + // Command prefixes + /// + /// Prefix for simple string responses (+) + /// + public const char STRING_MESSAGE = '+'; + + /// + /// Prefix for error responses (-) + /// + public const char ERROR_MESSAGE = '-'; + + /// + /// Prefix for integer responses (:) + /// + public const char INTEGER_MESSAGE = ':'; + + /// + /// Prefix for bulk string responses ($) + /// + public const char BULK_MESSAGE = '$'; + + /// + /// Prefix for array responses (*) + /// + public const char ARRAY_MESSAGE = '*'; + + /// + /// Special message indicating a nil value ($-1) + /// + public const string NIL_MESSAGE = "$-1"; + + /// + /// Standard message terminator sequence (\r\n) + /// + public const string MESSAGE_TERMINATOR = "\r\n"; + + /// + /// Standard OK response (+OK\r\n) + /// + public const string OK_RESPONSE = "+OK\r\n"; + + /// + /// Response for queued pipeline commands (+QUEUED\r\n) + /// + public const string QUEUED_RESPONSE = "+QUEUED\r\n"; + + /// + /// Standard nil response ($-1\r\n) + /// + public const string NIL_RESPONSE = "$-1\r\n"; + + /// + /// Command format specification for external clients + /// + public static class Commands + { + // Authentication + /// + /// Authenticate with the server using a password + /// + public const string AUTH = "AUTH {password}"; + + /// + /// Test server connection + /// + public const string PING = "PING"; + + /// + /// Close the connection + /// + public const string QUIT = "QUIT"; + + // String operations + /// + /// Set a key to hold a string value + /// + public const string SET = "SET {key} {value}"; + + /// + /// Get the value of a key + /// + public const string GET = "GET {key}"; + + /// + /// Delete a key + /// + public const string DEL = "DEL {key}"; + + // List operations + /// + /// Insert elements at the head of a list + /// + public const string LPUSH = "LPUSH {key} {value} [value...]"; + + /// + /// Insert elements at the tail of a list + /// + public const string RPUSH = "RPUSH {key} {value} [value...]"; + + /// + /// Remove and get the first element in a list + /// + public const string LPOP = "LPOP {key}"; + + /// + /// Remove and get the last element in a list + /// + public const string RPOP = "RPOP {key}"; + + /// + /// Get a range of elements from a list + /// + public const string LRANGE = "LRANGE {key} {start} {stop}"; + + // Hash operations + /// + /// Set the string value of a hash field + /// + public const string HSET = "HSET {key} {field} {value}"; + + /// + /// Get the value of a hash field + /// + public const string HGET = "HGET {key} {field}"; + + /// + /// Delete a hash field + /// + public const string HDEL = "HDEL {key} {field}"; + + /// + /// Get all fields and values in a hash + /// + public const string HGETALL = "HGETALL {key}"; + + // Pipeline operations + /// + /// Start a pipeline for batch processing + /// + public const string PIPELINE = "PIPELINE"; + + /// + /// Execute all commands in the pipeline + /// + public const string EXEC = "EXEC"; + } + + /// + /// Response format specification for external clients + /// + public static class Responses + { + /// + /// Format a simple string response + /// + /// The string value to format + /// Formatted string response + public static string FormatString(string value) => $"+{value}{MESSAGE_TERMINATOR}"; + + /// + /// Format an error response + /// + /// The error message + /// Formatted error response + public static string FormatError(string message) => $"-ERR {message}{MESSAGE_TERMINATOR}"; + + /// + /// Format an integer response + /// + /// The integer value to format + /// Formatted integer response + public static string FormatInteger(int value) => $":{value}{MESSAGE_TERMINATOR}"; + + /// + /// Format a bulk string response + /// + /// The string value to format + /// Formatted bulk string response + public static string FormatBulkString(string value) => $"${value.Length}{MESSAGE_TERMINATOR}{value}{MESSAGE_TERMINATOR}"; + + /// + /// Format an array response + /// + /// Array of strings to format + /// Formatted array response + public static string FormatArray(string[] values) + { + var result = $"*{values.Length}{MESSAGE_TERMINATOR}"; + foreach (var value in values) + { + result += FormatBulkString(value); + } + return result; + } + } + } +} \ No newline at end of file diff --git a/src/ServerManager.cs b/src/ServerManager.cs new file mode 100644 index 0000000..9b28f5c --- /dev/null +++ b/src/ServerManager.cs @@ -0,0 +1,310 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace Firefly +{ + public partial class Firefly + { + #region Server Management + static async Task StartServerAsync() + { + // Set up server on specified port + var listener = new TcpListener(IPAddress.Parse(bindAddress), serverPort); + listener.Start(); + Console.WriteLine($"Server is listening on {bindAddress}, port {serverPort}"); + + try + { + while (true) + { + // Accept client connection + var client = await listener.AcceptTcpClientAsync(); + + // Configure client + client.ReceiveBufferSize = bufferSize; + client.SendBufferSize = bufferSize; + client.ReceiveTimeout = connectionTimeoutSeconds * 1000; + client.SendTimeout = connectionTimeoutSeconds * 1000; + + Console.WriteLine("Client connected!"); + + // Handle client in a separate task + _ = HandleClientAsync(client); + } + } + catch (Exception ex) + { + Console.WriteLine($"Server error: {ex.Message}"); + } + finally + { + listener.Stop(); + } + } + + static async Task HandleClientAsync(TcpClient client) + { + string clientId = client.Client.RemoteEndPoint?.ToString() ?? Guid.NewGuid().ToString(); + + try + { + using (client) + using (var stream = client.GetStream()) + { + var buffer = new byte[bufferSize]; + var lastActivity = DateTime.Now; + var commandQueue = new Queue(); + clientCommandQueues[clientId] = commandQueue; + clientPipelineMode[clientId] = false; // Initialize pipeline mode to false + + while (client.Connected) + { + if ((DateTime.Now - lastActivity).TotalSeconds > connectionTimeoutSeconds) + { + Console.WriteLine($"Client {clientId} connection timed out due to inactivity"); + break; + } + + if (!stream.DataAvailable) + { + await Task.Delay(1); + continue; + } + + int bytesRead = await stream.ReadAsync(buffer); + if (bytesRead <= 0) + break; + + lastActivity = DateTime.Now; + var message = Encoding.UTF8.GetString(buffer, 0, bytesRead); + + // Split message into individual commands + var commands = message.Split(["\r\n"], StringSplitOptions.RemoveEmptyEntries); + + foreach (var cmd in commands) + { + if (string.IsNullOrWhiteSpace(cmd)) continue; + + // Check if this is a QUIT command + if (cmd.Trim().Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + await ProcessCommandBatch(clientId, commandQueue, stream); + await stream.WriteAsync(Encoding.UTF8.GetBytes("+OK\r\n")); + Console.WriteLine($"Client {clientId} requested to quit"); + return; + } + + // Check if this is a PIPELINE command + if (cmd.Trim().Equals("PIPELINE", StringComparison.OrdinalIgnoreCase)) + { + clientPipelineMode[clientId] = true; + await stream.WriteAsync(Encoding.UTF8.GetBytes("+OK\r\n")); + Console.WriteLine($"Client {clientId} entered pipeline mode"); + continue; + } + + // Check if this is an EXEC command (end pipeline mode) + if (cmd.Trim().Equals("EXEC", StringComparison.OrdinalIgnoreCase)) + { + clientPipelineMode[clientId] = false; + await ProcessCommandBatch(clientId, commandQueue, stream); + await stream.WriteAsync(Encoding.UTF8.GetBytes("+OK\r\n")); + Console.WriteLine($"Client {clientId} executed pipeline"); + continue; + } + + commandQueue.Enqueue(cmd); + + // Process commands if: + // 1. We've reached batch size + // 2. This is a non-pipeline command + // 3. We've reached max pipeline size + // 4. Client is not in pipeline mode + if (commandQueue.Count >= maxBatchSize || + IsNonPipelineCommand(cmd) || + commandQueue.Count >= maxPipelineSize || + !clientPipelineMode[clientId]) + { + await ProcessCommandBatch(clientId, commandQueue, stream); + } + } + } + } + } + catch (Exception ex) + { + if (ex is SocketException socketEx && + (socketEx.SocketErrorCode == SocketError.ConnectionReset || + socketEx.SocketErrorCode == SocketError.ConnectionAborted)) + { + Console.WriteLine($"Client {clientId} disconnected unexpectedly"); + } + else + { + Console.WriteLine($"Error handling client {clientId}: {ex.Message}"); + } + } + finally + { + clientCommandQueues.TryRemove(clientId, out _); + clientPipelineMode.TryRemove(clientId, out _); + authenticatedClients.Remove(clientId); + Console.WriteLine($"Client {clientId} disconnected."); + } + } + + private static bool IsNonPipelineCommand(string command) + { + var cmd = command.Trim().Split(' ')[0].ToUpperInvariant(); + return cmd == "AUTH" || cmd == "PING" || cmd == "QUIT" || cmd == "PIPELINE" || cmd == "EXEC"; + } + + private static string GetCommandType(string command) + { + var cmd = command.Trim().Split(' ')[0].ToUpperInvariant(); + + // Categorize commands by data type they work with + if (cmd == "SET" || cmd == "GET" || cmd == "DEL") + return "string"; + else if (cmd.StartsWith('L') || cmd == "RPUSH" || cmd == "RPOP") + return "list"; + else if (cmd.StartsWith('H')) + return "hash"; + else + return "other"; // Commands that don't fit clearly into a category + } + + private static async Task ProcessCommandBatch(string clientId, Queue commandQueue, NetworkStream stream) + { + if (commandQueue.Count == 0) return; + + var responses = new List(); + var batchSize = Math.Min(commandQueue.Count, maxBatchSize); + var isPipelineMode = clientPipelineMode.TryGetValue(clientId, out bool pipelineMode) && pipelineMode; + + if (isPipelineMode) + { + // Group commands by type when in pipeline mode + var stringCommands = new List(); + var listCommands = new List(); + var hashCommands = new List(); + var otherCommands = new List(); + + // Extract all commands from the queue and categorize + int commandsToProcess = Math.Min(commandQueue.Count, maxBatchSize); + for (int i = 0; i < commandsToProcess; i++) + { + if (commandQueue.Count == 0) break; + string command = commandQueue.Dequeue(); + + string commandType = GetCommandType(command); + switch (commandType) + { + case "string": + stringCommands.Add(command); + break; + case "list": + listCommands.Add(command); + break; + case "hash": + hashCommands.Add(command); + break; + default: + otherCommands.Add(command); + break; + } + } + + // Process each category in separate batches + var allResults = new List(); + + // Process string commands + if (stringCommands.Count > 0) + { + var stringTasks = stringCommands.Select(cmd => Task.Run(() => ProcessCommand(cmd, clientId))).ToList(); + var stringResults = await Task.WhenAll(stringTasks); + allResults.AddRange(stringResults); + Console.WriteLine($"Processed {stringCommands.Count} string commands for client {clientId}"); + } + + // Process list commands + if (listCommands.Count > 0) + { + var listTasks = listCommands.Select(cmd => Task.Run(() => ProcessCommand(cmd, clientId))).ToList(); + var listResults = await Task.WhenAll(listTasks); + allResults.AddRange(listResults); + Console.WriteLine($"Processed {listCommands.Count} list commands for client {clientId}"); + } + + // Process hash commands + if (hashCommands.Count > 0) + { + var hashTasks = hashCommands.Select(cmd => Task.Run(() => ProcessCommand(cmd, clientId))).ToList(); + var hashResults = await Task.WhenAll(hashTasks); + allResults.AddRange(hashResults); + Console.WriteLine($"Processed {hashCommands.Count} hash commands for client {clientId}"); + } + + // Process other commands + if (otherCommands.Count > 0) + { + var otherTasks = otherCommands.Select(cmd => Task.Run(() => ProcessCommand(cmd, clientId))).ToList(); + var otherResults = await Task.WhenAll(otherTasks); + allResults.AddRange(otherResults); + Console.WriteLine($"Processed {otherCommands.Count} other commands for client {clientId}"); + } + + responses = allResults; + } + else + { + // Non-pipeline mode - process in original order as before + var tasks = new List>(); + var commands = new List(); + + // Extract commands to process + for (int i = 0; i < batchSize; i++) + { + if (commandQueue.Count == 0) break; + commands.Add(commandQueue.Dequeue()); + } + + // Process commands in parallel + foreach (var command in commands) + { + tasks.Add(Task.Run(() => ProcessCommand(command, clientId))); + } + + // Wait for all tasks to complete + var results = await Task.WhenAll(tasks); + responses.AddRange(results); + } + + // Send all responses in a single write + if (responses.Count > 0) + { + var combinedResponse = new byte[responses.Sum(r => r.Length)]; + var offset = 0; + foreach (var response in responses) + { + Buffer.BlockCopy(response, 0, combinedResponse, offset, response.Length); + offset += response.Length; + } + + await stream.WriteAsync(combinedResponse); + + // Log batch processing information + if (isPipelineMode) + { + Console.WriteLine($"Processed grouped pipeline batch of {responses.Count} commands for client {clientId}"); + } + else if (responses.Count > 1) + { + Console.WriteLine($"Processed batch of {responses.Count} commands for client {clientId}"); + } + } + } + #endregion + } +} \ No newline at end of file diff --git a/src/ServerOperations.cs b/src/ServerOperations.cs new file mode 100644 index 0000000..761acfb --- /dev/null +++ b/src/ServerOperations.cs @@ -0,0 +1,96 @@ +using System.Text; + +namespace Firefly +{ + public partial class Firefly + { + #region Server Operations + /// + /// Handles the AUTH command which authenticates a client. + /// + /// The password to authenticate with + /// The client ID + /// OK on success, error if authentication fails + static byte[] HandleAuthCommand(string password, string clientId) + { + if (string.IsNullOrEmpty(serverPassword)) + { + Console.WriteLine("AUTH command received but server has no password set"); + return Encoding.UTF8.GetBytes("-ERR this server does not require authentication\r\n"); + } + + if (serverPassword == password) + { + Console.WriteLine($"Client {clientId} successfully authenticated"); + authenticatedClients[clientId] = true; + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + Console.WriteLine($"Client {clientId} failed authentication attempt with password: {password}"); + return Encoding.UTF8.GetBytes("-ERR invalid password\r\n"); + } + + /// + /// Checks if a client is authenticated. + /// + /// The client ID + /// True if the client is authenticated, false otherwise + static bool IsAuthenticated(string clientId) + { + // If no password is set, all clients are authenticated by default + if (string.IsNullOrEmpty(serverPassword)) + { + return true; + } + + return authenticatedClients.TryGetValue(clientId, out bool authenticated) && authenticated; + } + + /// + /// Handles the SAVE command which triggers a manual backup of the data. + /// + static byte[] HandleSaveCommand() + { + try + { + BackupData(); + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + catch (Exception ex) + { + return Encoding.UTF8.GetBytes($"-ERR save failed: {ex.Message}\r\n"); + } + } + + /// + /// Handles the BGSAVE command which triggers an asynchronous backup of the data. + /// + static byte[] HandleBgSaveCommand() + { + // Start backup in a separate task + Task.Run(() => BackupData()); + return Encoding.UTF8.GetBytes("+Background saving started\r\n"); + } + + /// + /// Handles server shutdown operations + /// + static async Task ShutdownServer() + { + if (backupsEnabled) + { + Console.WriteLine("Performing final backup before shutdown..."); + await Task.Run(() => BackupData()); + } + + // Clean up any remaining client connections + foreach (var clientId in authenticatedClients.Keys.ToList()) + { + authenticatedClients.Remove(clientId); + } + + Console.WriteLine("Server shutdown complete."); + } + #endregion + } +} \ No newline at end of file diff --git a/src/StringOperations.cs b/src/StringOperations.cs new file mode 100644 index 0000000..aad07f1 --- /dev/null +++ b/src/StringOperations.cs @@ -0,0 +1,119 @@ +using System.Text; + +namespace Firefly +{ + public partial class Firefly + { + #region String Operations + /// + /// Handles the SET command which sets a key-value pair in the string store. + /// + /// Command arguments in format: "key value" + /// OK on success, error if arguments are invalid + static byte[] HandleSetCommand(string args) + { + string[] parts = args.Split(' ', 2); + if (parts.Length < 2) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'set' command\r\n"); + } + + string key = parts[0]; + string value = parts[1]; + + // Store the key-value pair using the sharded helper method + if (!StringStoreSet(key, value)) + { + string? existingType = GetKeyType(key); + if (existingType != null) + { + return Encoding.UTF8.GetBytes($"-ERR key '{key}' already exists as type '{existingType}'\r\n"); + } + return Encoding.UTF8.GetBytes("-ERR operation failed\r\n"); + } + + return Encoding.UTF8.GetBytes("+OK\r\n"); + } + + /// + /// Handles the GET command which retrieves a value from the string store. + /// + /// Command arguments containing the key + /// The value associated with the key, or nil if the key doesn't exist + static byte[] HandleGetCommand(string args) + { + if (string.IsNullOrWhiteSpace(args)) + { + return Encoding.UTF8.GetBytes("-ERR wrong number of arguments for 'get' command\r\n"); + } + + string key = args.Trim(); + + // Retrieve the value using the sharded helper method + if (StringStoreGet(key, out string? value)) + { + return Encoding.UTF8.GetBytes($"+{value}\r\n"); + } + else + { + return Encoding.UTF8.GetBytes("$-1\r\n"); // Redis returns nil for non-existent keys + } + } + + #region String Store Helpers + /// + /// Checks if a string exists for a given key. + /// + /// The key to check the string for + /// True if the string exists, false otherwise + private static bool StringStoreExists(string key) + { + int shardIndex = GetShardIndex(key); + return stringStoreShards[shardIndex].ContainsKey(key); + } + + /// + /// Sets a string for a given key. + /// + /// The key to set the string for + /// The value to set + /// True if the string was set, false otherwise + private static bool StringStoreSet(string key, string value) + { + // Check if key exists in any other store + if (!EnsureKeyDoesNotExist(key, "string")) + { + return false; + } + + int shardIndex = GetShardIndex(key); + stringStoreShards[shardIndex][key] = value; + return true; + } + + /// + /// Gets a string for a given key. + /// + /// The key to get the string for + /// The value + /// True if the string was found, false otherwise + private static bool StringStoreGet(string key, out string? value) + { + int shardIndex = GetShardIndex(key); + return stringStoreShards[shardIndex].TryGetValue(key, out value); + } + + /// + /// Removes a string for a given key. + /// + /// The key to remove the string for + /// True if the string was removed, false otherwise + private static bool StringStoreRemove(string key) + { + int shardIndex = GetShardIndex(key); + return stringStoreShards[shardIndex].TryRemove(key, out _); + } + #endregion + #endregion + } +} \ No newline at end of file