SNote/Server/Endpoints/SyncEndpoints.cs

using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using SNote.Models;
using SNote.Server.Data;
using SNote.Server.Security;

namespace SNote.Server.Endpoints;

public static class SyncEndpoints
{
    public static string ServerGuid { get; } = Guid.NewGuid().ToString();
    public static string? UpstreamServerGuid { get; set; } = null;
    public static string? UpstreamSessionToken { get; set; } = null;
    public static void MapSyncEndpoints(this IEndpointRouteBuilder routes)
    {
        // 0. Heartbeat ping endpoint: downstream returns its public key, current timestamp, and a signature (token) of the timestamp
        routes.MapGet("/api/sync/ping", (RsaKeyManager rsaKeyManager) =>
        {
            try
            {
                var timestamp = DateTime.UtcNow.ToString("o");
                var dataToSign = System.Text.Encoding.UTF8.GetBytes(timestamp);

                var privateRsa = rsaKeyManager.GetPrivateKey();
                var signatureBytes = privateRsa.SignData(dataToSign, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1);
                var token = Convert.ToBase64String(signatureBytes);

                return Results.Ok(new
                {
                    publicKey = rsaKeyManager.GetPublicKeyPem(),
                    timestamp = timestamp,
                    token = token
                });
            }
            catch (Exception ex)
            {
                return Results.Json(new { error = $"Heartbeat generation failed: {ex.Message}" }, statusCode: 500);
            }
        });

        // 1. Handshake Endpoint: Receive registration of downstream peers with RSA asymmetric validation
        routes.MapPost("/api/sync/register-peer", async (RegisterPeerRequest req, HttpContext context, PeerCache peerCache, RsaKeyManager rsaKeyManager) =>
        {
            if (string.IsNullOrWhiteSpace(req.PeerGuid) || string.IsNullOrWhiteSpace(req.PeerUrl) || 
                string.IsNullOrWhiteSpace(req.EncryptedSecret) || string.IsNullOrWhiteSpace(req.PublicKey) || 
                string.IsNullOrWhiteSpace(req.Signature))
            {
                return Results.BadRequest("Missing required handshake parameters.");
            }

            try
            {
                // 1. Decrypt the secret using our Private Key
                var localPrivateRsa = rsaKeyManager.GetPrivateKey();
                var encryptedBytes = Convert.FromBase64String(req.EncryptedSecret);
                var decryptedBytes = localPrivateRsa.Decrypt(encryptedBytes, System.Security.Cryptography.RSAEncryptionPadding.OaepSHA256);
                var secret = System.Text.Encoding.UTF8.GetString(decryptedBytes);

                // 2. Verify signature using downstream's Public Key
                using var peerPublicRsa = System.Security.Cryptography.RSA.Create();
                peerPublicRsa.ImportFromPem(req.PublicKey);

                var dataToVerify = System.Text.Encoding.UTF8.GetBytes($"{req.PeerGuid}|{secret}");
                var signatureBytes = Convert.FromBase64String(req.Signature);
                var verified = peerPublicRsa.VerifyData(dataToVerify, signatureBytes, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1);

                if (!verified)
                {
                    return Results.Json(new { error = "Asymmetric signature verification failed." }, statusCode: 401);
                }

                // 3. Issue secure session token and register peer details
                var sessionToken = Guid.NewGuid().ToString();
                peerCache.RegisterPeer(req.PeerGuid, sessionToken, req.PublicKey, req.PeerUrl);

                Console.WriteLine($"[Handshake] Successfully validated child server '{req.PeerGuid}' (URL: '{req.PeerUrl}') via RSA and issued token.");
                return Results.Ok(new HandshakeResponse(sessionToken, ServerGuid));
            }
            catch (Exception ex)
            {
                Console.WriteLine($"[Handshake] Handshake registration error: {ex.Message}");
                return Results.Json(new { error = $"Handshake validation error: {ex.Message}" }, statusCode: 400);
            }
        });

        // 2. Full Database pulling (bootstrapping new server node)
        routes.MapPost("/api/sync/pull-database", async (HttpContext context, ServerDbContext db, IConfiguration config, PeerCache peerCache) =>
        {
            var enablePull = config.GetValue<bool>("EnableFullDbPull", false);
            if (!enablePull)
            {
                return Results.Json(new { error = "Full database pulling is disabled on this server." }, statusCode: 403);
            }

            if (!AuthHelper.IsServerTokenValid(context, peerCache))
            {
                return Results.Json(new { error = "Unauthorized server token." }, statusCode: 401);
            }

            var users = await db.Users.ToListAsync();
            var nodes = await db.Nodes.ToListAsync();
            var nodeKeys = await db.NodeKeys.ToListAsync();
            var pendingShares = await db.PendingShares.ToListAsync();
            var aclEntries = await db.AclEntries.ToListAsync();

            var dump = new DatabaseDumpPayload(users, nodes, nodeKeys, pendingShares, aclEntries);
            return Results.Ok(dump);
        });

        // 3. Synchronize nodes states
        routes.MapPost("/api/sync/nodes", async (SyncExchangeRequest req, HttpContext context, ServerDbContext db, PeerCache peerCache) =>
        {
            if (!AuthHelper.IsServerTokenValid(context, peerCache))
            {
                return Results.Unauthorized();
            }

            var toPush = new List<SyncPushItem>();
            var toRequest = new List<string>();

            var localNodes = await db.Nodes.ToListAsync();
            var localNodeUuids = localNodes.Select(n => n.Uuid).ToHashSet();

            foreach (var remoteItem in req.Items)
            {
                var localNode = localNodes.FirstOrDefault(n => n.Uuid == remoteItem.Uuid);
                if (localNode == null)
                {
                    toRequest.Add(remoteItem.Uuid);
                }
                else
                {
                    if (localNode.UpdatedAt > remoteItem.UpdatedAt)
                    {
                        var acls = await db.AclEntries.Where(a => a.NodeUuid == localNode.Uuid).ToListAsync();
                        var keys = await db.NodeKeys.Where(k => k.NodeUuid == localNode.Uuid).ToListAsync();
                        toPush.Add(new SyncPushItem(localNode, acls, keys));
                    }
                    else if (localNode.UpdatedAt < remoteItem.UpdatedAt)
                    {
                        toRequest.Add(remoteItem.Uuid);
                    }
                }
            }

            var remoteUuids = req.Items.Select(i => i.Uuid).ToHashSet();
            foreach (var localNode in localNodes)
            {
                if (!remoteUuids.Contains(localNode.Uuid))
                {
                    var acls = await db.AclEntries.Where(a => a.NodeUuid == localNode.Uuid).ToListAsync();
                    var keys = await db.NodeKeys.Where(k => k.NodeUuid == localNode.Uuid).ToListAsync();
                    toPush.Add(new SyncPushItem(localNode, acls, keys));
                }
            }

            return Results.Ok(new SyncExchangeResponse(toPush, toRequest));
        });

        // 4. Receive full node pushes during sync
        routes.MapPost("/api/sync/push", async (List<SyncPushItem> items, HttpContext context, ServerDbContext db, PeerCache peerCache) =>
        {
            if (!AuthHelper.IsServerTokenValid(context, peerCache))
            {
                return Results.Unauthorized();
            }

            foreach (var item in items)
            {
                var localNode = await db.Nodes.FirstOrDefaultAsync(n => n.Uuid == item.Node.Uuid);
                if (localNode == null)
                {
                    db.Nodes.Add(item.Node);
                    db.AclEntries.AddRange(item.AclEntries);
                    db.NodeKeys.AddRange(item.NodeKeys);
                }
                else if (localNode.UpdatedAt < item.Node.UpdatedAt)
                {
                    localNode.Name = item.Node.Name;
                    localNode.Type = item.Node.Type;
                    localNode.Content = item.Node.Content;
                    localNode.OwnerUsername = item.Node.OwnerUsername;
                    localNode.UpdatedAt = item.Node.UpdatedAt;
                    localNode.IsDeleted = item.Node.IsDeleted;

                    var existingAcls = await db.AclEntries.Where(a => a.NodeUuid == item.Node.Uuid).ToListAsync();
                    db.AclEntries.RemoveRange(existingAcls);
                    db.AclEntries.AddRange(item.AclEntries);

                    var existingKeys = await db.NodeKeys.Where(k => k.NodeUuid == item.Node.Uuid).ToListAsync();
                    db.NodeKeys.RemoveRange(existingKeys);
                    db.NodeKeys.AddRange(item.NodeKeys);
                }
            }

            await db.SaveChangesAsync();
            return Results.Ok(new { message = "Synced elements updated successfully." });
        });

        // 5. Receive broadcast of changes (broadcasts single node updates)
        routes.MapPost("/api/sync/broadcast", async (SyncPushItem item, HttpContext context, ServerDbContext db, PeerCache peerCache, RsaKeyManager rsaKeyManager, HttpClient httpClient, IServiceProvider serviceProvider) =>
        {
            if (!AuthHelper.IsServerTokenValid(context, peerCache))
            {
                return Results.Unauthorized();
            }

            var broadcastId = item.BroadcastId;
            if (!string.IsNullOrEmpty(broadcastId))
            {
                // Loop check: if already processed, ignore!
                if (!peerCache.TryProcessBroadcast(broadcastId))
                {
                    return Results.Ok(new { message = "Broadcast already processed (loop detected).", updated = false });
                }
            }

            var localNode = await db.Nodes.FirstOrDefaultAsync(n => n.Uuid == item.Node.Uuid);
            bool updated = false;

            if (localNode == null)
            {
                db.Nodes.Add(item.Node);
                db.AclEntries.AddRange(item.AclEntries);
                db.NodeKeys.AddRange(item.NodeKeys);
                if (item.PendingShares != null)
                {
                    db.PendingShares.AddRange(item.PendingShares);
                }
                updated = true;
            }
            else if (localNode.UpdatedAt < item.Node.UpdatedAt)
            {
                localNode.Name = item.Node.Name;
                localNode.Type = item.Node.Type;
                localNode.Content = item.Node.Content;
                localNode.OwnerUsername = item.Node.OwnerUsername;
                localNode.UpdatedAt = item.Node.UpdatedAt;
                localNode.IsDeleted = item.Node.IsDeleted;

                var existingAcls = await db.AclEntries.Where(a => a.NodeUuid == item.Node.Uuid).ToListAsync();
                db.AclEntries.RemoveRange(existingAcls);
                db.AclEntries.AddRange(item.AclEntries);

                var existingKeys = await db.NodeKeys.Where(k => k.NodeUuid == item.Node.Uuid).ToListAsync();
                db.NodeKeys.RemoveRange(existingKeys);
                db.NodeKeys.AddRange(item.NodeKeys);

                var existingShares = await db.PendingShares.Where(s => s.NodeUuid == item.Node.Uuid).ToListAsync();
                db.PendingShares.RemoveRange(existingShares);
                if (item.PendingShares != null)
                {
                    db.PendingShares.AddRange(item.PendingShares);
                }

                updated = true;
            }

            if (updated)
            {
                await db.SaveChangesAsync();

                if (!string.IsNullOrEmpty(broadcastId))
                {
                    _ = Task.Run(async () =>
                    {
                        try
                        {
                            using var scope = serviceProvider.CreateScope();
                            var backgroundDb = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
                            var backgroundConfig = scope.ServiceProvider.GetRequiredService<IConfiguration>();
                            await BroadcastNodeChangeAsync(item.Node.Uuid, backgroundDb, peerCache, rsaKeyManager, backgroundConfig, httpClient, broadcastId);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine($"Error during background BroadcastNodeChangeAsync: {ex.Message}");
                        }
                    });
                }
            }

            return Results.Ok(new { message = "Broadcast processed.", updated });
        });

        // Receive broadcast of user registration or password updates
        routes.MapPost("/api/sync/broadcast-user", async (UserBroadcastPayload payload, HttpContext context, ServerDbContext db, PeerCache peerCache, RsaKeyManager rsaKeyManager, HttpClient httpClient, IServiceProvider serviceProvider) =>
        {
            if (!AuthHelper.IsServerTokenValid(context, peerCache))
            {
                return Results.Unauthorized();
            }

            var broadcastId = payload.BroadcastId;
            if (!string.IsNullOrEmpty(broadcastId))
            {
                if (!peerCache.TryProcessBroadcast(broadcastId))
                {
                    return Results.Ok(new { message = "User broadcast already processed (loop detected).", updated = false });
                }
            }

            var existingUser = await db.Users.FirstOrDefaultAsync(u => u.Username.ToLower() == payload.User.Username.ToLower());
            bool updated = false;

            if (existingUser == null)
            {
                db.Users.Add(payload.User);
                updated = true;
            }
            else
            {
                if (existingUser.PasswordHash != payload.User.PasswordHash || existingUser.EncryptedKey != payload.User.EncryptedKey)
                {
                    existingUser.PasswordHash = payload.User.PasswordHash;
                    existingUser.EncryptedKey = payload.User.EncryptedKey;
                    updated = true;
                }
            }

            if (updated)
            {
                await db.SaveChangesAsync();

                if (!string.IsNullOrEmpty(broadcastId))
                {
                    _ = Task.Run(async () =>
                    {
                        try
                        {
                            using var scope = serviceProvider.CreateScope();
                            var backgroundDb = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
                            var backgroundConfig = scope.ServiceProvider.GetRequiredService<IConfiguration>();
                            await BroadcastUserChangeAsync(payload.User.Username, backgroundDb, peerCache, rsaKeyManager, backgroundConfig, httpClient, broadcastId);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine($"Error during background BroadcastUserChangeAsync: {ex.Message}");
                        }
                    });
                }
            }

            return Results.Ok(new { message = "User broadcast processed.", updated });
        });
    }

    // Handshake: Tells the destination server who we are and registers us using RSA asymmetric validation
    public static async Task RegisterPeerWithDestinationAsync(string destinationUrl, string localUrl, RsaKeyManager rsaKeyManager, IConfiguration configuration, HttpClient httpClient)
    {
        try
        {
            var cleanDestUrl = (destinationUrl ?? "").Trim().TrimEnd('/');
            var cleanLocalUrl = (localUrl ?? "").Trim().TrimEnd('/');
            Console.WriteLine($"[Handshake] Performing RSA handshake with destination '{cleanDestUrl}' using local url '{cleanLocalUrl}'...");
            
            // 1. Get public key from RsaKeyManager
            var upstreamPublicKey = rsaKeyManager.GetPublicKeyPem();
            if (string.IsNullOrEmpty(upstreamPublicKey))
            {
                Console.WriteLine("[Handshake] Error: Public key is not loaded in RsaKeyManager. RSA handshake cannot proceed.");
                return;
            }

            var cleanDest = cleanDestUrl + "/api/sync/register-peer";

            // 2. Generate secret challenge
            var secret = Guid.NewGuid().ToString();

            // 3. Encrypt secret challenge using upstream's public key
            using var upstreamRsa = System.Security.Cryptography.RSA.Create();
            upstreamRsa.ImportFromPem(upstreamPublicKey);
            
            var secretBytes = System.Text.Encoding.UTF8.GetBytes(secret);
            var encryptedBytes = upstreamRsa.Encrypt(secretBytes, System.Security.Cryptography.RSAEncryptionPadding.OaepSHA256);
            var encryptedSecretBase64 = Convert.ToBase64String(encryptedBytes);

            // 4. Sign payload (ServerGuid + secret) using our own private key
            var localPrivateRsa = rsaKeyManager.GetPrivateKey();
            var dataToSign = System.Text.Encoding.UTF8.GetBytes($"{ServerGuid}|{secret}");
            var signatureBytes = localPrivateRsa.SignData(dataToSign, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1);
            var signatureBase64 = Convert.ToBase64String(signatureBytes);

            // 5. Send registration handshake request
            var payload = new RegisterPeerRequest(ServerGuid, cleanLocalUrl, encryptedSecretBase64, rsaKeyManager.GetPublicKeyPem(), signatureBase64);
            var json = JsonSerializer.Serialize(payload);
            var request = new HttpRequestMessage(HttpMethod.Post, cleanDest)
            {
                Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json")
            };

            var response = await httpClient.SendAsync(request);
            if (response.IsSuccessStatusCode)
            {
                var responseContent = await response.Content.ReadAsStringAsync();
                var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
                var responseData = JsonSerializer.Deserialize<HandshakeResponse>(responseContent, options);

                if (responseData != null && !string.IsNullOrEmpty(responseData.SessionToken))
                {
                    UpstreamSessionToken = responseData.SessionToken;
                    UpstreamServerGuid = responseData.ServerGuid;
                    Console.WriteLine($"[Handshake] Successfully handshaked and obtained Session Token: {UpstreamSessionToken}, Upstream GUID: {UpstreamServerGuid}");
                }
                else
                {
                    Console.WriteLine("[Handshake] Handshake warning: Handshake returned success but no session token/GUID was found.");
                }
            }
            else
            {
                Console.WriteLine($"[Handshake] Handshake failed with status: {response.StatusCode}");
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"[Handshake] Handshake error: {ex.Message}");
        }
    }

    // Broadcast a node change to both database and dynamic in-memory cached peer servers with token security
    public static async Task BroadcastNodeChangeAsync(string nodeUuid, ServerDbContext db, PeerCache peerCache, RsaKeyManager rsaKeyManager, IConfiguration configuration, HttpClient httpClient, string? broadcastId = null)
    {
        var node = await db.Nodes.FirstOrDefaultAsync(n => n.Uuid == nodeUuid);
        if (node == null) return;

        var acls = await db.AclEntries.Where(a => a.NodeUuid == nodeUuid).ToListAsync();
        var keys = await db.NodeKeys.Where(k => k.NodeUuid == nodeUuid).ToListAsync();
        var pendingShares = await db.PendingShares.Where(s => s.NodeUuid == nodeUuid).ToListAsync();

        var bId = string.IsNullOrEmpty(broadcastId) ? Guid.NewGuid().ToString() : broadcastId;
        
        // Register it locally in PeerCache so we don't process it ourselves if it loops back
        peerCache.TryProcessBroadcast(bId);

        var payload = new SyncPushItem(node, acls, keys, bId, pendingShares);
        var json = JsonSerializer.Serialize(payload);

        // Fetch dynamically handshake-registered in-memory peers (downstream child nodes) and upstream
        var targets = new List<BroadcastTarget>();
        foreach (var peer in peerCache.GetActivePeers())
        {
            targets.Add(new BroadcastTarget(peer.Guid, peer.Url, peer.Token, false));
        }

        var destUrl = (configuration["Sync:DestinationServerUrl"] ?? "").Trim().TrimEnd('/');
        var localUrl = (configuration["Sync:LocalServerUrl"] ?? "").Trim().TrimEnd('/');

        if (!string.IsNullOrEmpty(destUrl))
        {
            if (!targets.Any(t => string.Equals(t.Url, destUrl, StringComparison.OrdinalIgnoreCase)))
            {
                targets.Add(new BroadcastTarget(UpstreamServerGuid ?? "", destUrl, UpstreamSessionToken ?? "", true));
            }
        }

        foreach (var target in targets)
        {
            try
            {
                var request = new HttpRequestMessage(HttpMethod.Post, $"{target.Url.TrimEnd('/')}/api/sync/broadcast");
                request.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");

                // Attach custom headers for secure authentication using our dynamic ServerGuid identity
                request.Headers.Add("X-Server-Id", ServerGuid);

                if (!string.IsNullOrEmpty(target.Token))
                {
                    request.Headers.Add("X-Server-Token", target.Token);
                }

                var response = await httpClient.SendAsync(request);
                Console.WriteLine($"Broadcast to {target.Url}: {response.StatusCode}");

                if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized && target.IsUpstream)
                {
                    Console.WriteLine($"[Sync] Received 401 Unauthorized from upstream destination '{destUrl}'. Initiating real-time re-handshake...");
                    
                    // Re-perform handshake to get a fresh UpstreamSessionToken
                    await RegisterPeerWithDestinationAsync(destUrl, localUrl, rsaKeyManager, configuration, httpClient);
                    
                    if (!string.IsNullOrEmpty(UpstreamSessionToken))
                    {
                        Console.WriteLine("[Sync] Re-handshake succeeded. Retrying upstream broadcast with new session token...");
                        
                        // Re-create the request to retry
                        var retryRequest = new HttpRequestMessage(HttpMethod.Post, $"{target.Url.TrimEnd('/')}/api/sync/broadcast");
                        retryRequest.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");
                        retryRequest.Headers.Add("X-Server-Id", ServerGuid);
                        retryRequest.Headers.Add("X-Server-Token", UpstreamSessionToken);
                        
                        var retryResponse = await httpClient.SendAsync(retryRequest);
                        Console.WriteLine($"Retried broadcast to {target.Url}: {retryResponse.StatusCode}");
                    }
                    else
                    {
                        Console.WriteLine("[Sync] Re-handshake failed. Unable to retry upstream broadcast.");
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Failed to broadcast change to {target.Url}: {ex.Message}");
            }
        }
    }

    public static async Task BroadcastUserChangeAsync(string username, ServerDbContext db, PeerCache peerCache, RsaKeyManager rsaKeyManager, IConfiguration configuration, HttpClient httpClient, string? broadcastId = null)
    {
        var user = await db.Users.FirstOrDefaultAsync(u => u.Username.ToLower() == username.ToLower());
        if (user == null) return;

        var bId = string.IsNullOrEmpty(broadcastId) ? Guid.NewGuid().ToString() : broadcastId;
        
        // Register it locally in PeerCache so we don't process it ourselves if it loops back
        peerCache.TryProcessBroadcast(bId);

        var payload = new UserBroadcastPayload(user, bId);
        var json = JsonSerializer.Serialize(payload);

        // Fetch dynamically handshake-registered in-memory peers (downstream child nodes) and upstream
        var targets = new List<BroadcastTarget>();
        foreach (var peer in peerCache.GetActivePeers())
        {
            targets.Add(new BroadcastTarget(peer.Guid, peer.Url, peer.Token, false));
        }

        var destUrl = (configuration["Sync:DestinationServerUrl"] ?? "").Trim().TrimEnd('/');
        var localUrl = (configuration["Sync:LocalServerUrl"] ?? "").Trim().TrimEnd('/');

        if (!string.IsNullOrEmpty(destUrl))
        {
            if (!targets.Any(t => string.Equals(t.Url, destUrl, StringComparison.OrdinalIgnoreCase)))
            {
                targets.Add(new BroadcastTarget(UpstreamServerGuid ?? "", destUrl, UpstreamSessionToken ?? "", true));
            }
        }

        foreach (var target in targets)
        {
            try
            {
                var request = new HttpRequestMessage(HttpMethod.Post, $"{target.Url.TrimEnd('/')}/api/sync/broadcast-user");
                request.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");

                // Attach custom headers for secure authentication using our dynamic ServerGuid identity
                request.Headers.Add("X-Server-Id", ServerGuid);

                if (!string.IsNullOrEmpty(target.Token))
                {
                    request.Headers.Add("X-Server-Token", target.Token);
                }

                var response = await httpClient.SendAsync(request);
                Console.WriteLine($"User broadcast to {target.Url}: {response.StatusCode}");

                if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized && target.IsUpstream)
                {
                    Console.WriteLine($"[Sync] Received 401 Unauthorized from upstream destination '{destUrl}'. Initiating real-time re-handshake for user broadcast...");
                    
                    // Re-perform handshake to get a fresh UpstreamSessionToken
                    await RegisterPeerWithDestinationAsync(destUrl, localUrl, rsaKeyManager, configuration, httpClient);
                    
                    if (!string.IsNullOrEmpty(UpstreamSessionToken))
                    {
                        Console.WriteLine("[Sync] Re-handshake succeeded. Retrying upstream user broadcast with new session token...");
                        
                        // Re-create the request to retry
                        var retryRequest = new HttpRequestMessage(HttpMethod.Post, $"{target.Url.TrimEnd('/')}/api/sync/broadcast-user");
                        retryRequest.Content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");
                        retryRequest.Headers.Add("X-Server-Id", ServerGuid);
                        retryRequest.Headers.Add("X-Server-Token", UpstreamSessionToken);
                        
                        var retryResponse = await httpClient.SendAsync(retryRequest);
                        Console.WriteLine($"Retried user broadcast to {target.Url}: {retryResponse.StatusCode}");
                    }
                    else
                    {
                        Console.WriteLine("[Sync] Re-handshake failed. Unable to retry upstream user broadcast.");
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Failed to broadcast user change to {target.Url}: {ex.Message}");
            }
        }
    }

    // Periodically pings downstream peer servers in PeerCache to ensure they are online and authentic
    public static async Task StartHeartbeatPingerAsync(PeerCache peerCache, RsaKeyManager rsaKeyManager, HttpClient httpClient, System.Threading.CancellationToken token)
    {
        Console.WriteLine("[Heartbeat] Starting periodic downstream heartbeat pinger...");
        while (!token.IsCancellationRequested)
        {
            try
            {
                // Wait for 30 seconds
                await Task.Delay(30000, token);

                var peers = peerCache.GetActivePeers();
                if (peers.Count == 0) continue;

                foreach (var peer in peers)
                {
                    try
                    {
                        var request = new HttpRequestMessage(HttpMethod.Get, $"{peer.Url.TrimEnd('/')}/api/sync/ping");
                        
                        // Set a short timeout (e.g. 5 seconds) to avoid blocking
                        using var cts = new System.Threading.CancellationTokenSource(TimeSpan.FromSeconds(5));
                        using var linkedCts = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(token, cts.Token);

                        var response = await httpClient.SendAsync(request, linkedCts.Token);
                        if (!response.IsSuccessStatusCode)
                        {
                            Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) ping failed with status code: {response.StatusCode}. Evicting...");
                            peerCache.RemovePeer(peer.Guid);
                            continue;
                        }

                        var responseContent = await response.Content.ReadAsStringAsync();
                        var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
                        var pingData = JsonSerializer.Deserialize<PingResponse>(responseContent, options);

                        if (pingData == null || string.IsNullOrEmpty(pingData.PublicKey) || 
                            string.IsNullOrEmpty(pingData.Timestamp) || string.IsNullOrEmpty(pingData.Token))
                        {
                            Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) returned invalid heartbeat response format. Evicting...");
                            peerCache.RemovePeer(peer.Guid);
                            continue;
                        }

                        // 1. Verify that the returned Public Key matches the registered Public Key in our cache!
                        var expectedPublicKey = peerCache.GetPublicKey(peer.Guid);
                        if (string.IsNullOrEmpty(expectedPublicKey) || 
                            !string.Equals(expectedPublicKey.Trim(), pingData.PublicKey.Trim(), StringComparison.Ordinal))
                        {
                            Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) public key mismatch! Security warning. Evicting...");
                            peerCache.RemovePeer(peer.Guid);
                            continue;
                        }

                        // 2. Verify signature ("token") over the timestamp
                        using var peerPublicRsa = System.Security.Cryptography.RSA.Create();
                        peerPublicRsa.ImportFromPem(pingData.PublicKey);

                        var dataToVerify = System.Text.Encoding.UTF8.GetBytes(pingData.Timestamp);
                        var signatureBytes = Convert.FromBase64String(pingData.Token);
                        var verified = peerPublicRsa.VerifyData(dataToVerify, signatureBytes, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1);

                        if (!verified)
                        {
                            Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) signature verification failed! Security warning. Evicting...");
                            peerCache.RemovePeer(peer.Guid);
                            continue;
                        }

                        // Heartbeat successful!
                        Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) is healthy and authentic.");
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"[Heartbeat] Peer {peer.Url} (GUID: {peer.Guid}) is unreachable ({ex.Message}). Evicting...");
                        peerCache.RemovePeer(peer.Guid);
                    }
                }
            }
            catch (TaskCanceledException)
            {
                break;
            }
            catch (Exception ex)
            {
                Console.WriteLine($"[Heartbeat] Error in heartbeat loop: {ex.Message}");
            }
        }
    }

    public static async Task SyncWithDestinationServerAsync(string peerUrl, ServerDbContext db, CertificateManager certManager, HttpClient httpClient)
    {
        try
        {
            var token = AuthHelper.GenerateServerToken(certManager);

            var localNodes = await db.Nodes.ToListAsync();
            var reqItems = localNodes.Select(n => new SyncNodeState(n.Uuid, n.UpdatedAt, n.IsDeleted)).ToList();

            var request = new HttpRequestMessage(HttpMethod.Post, $"{peerUrl.TrimEnd('/')}/api/sync/nodes");
            request.Content = new StringContent(JsonSerializer.Serialize(new SyncExchangeRequest(reqItems)), Encoding.UTF8, "application/json");
            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);

            var response = await httpClient.SendAsync(request);
            if (!response.IsSuccessStatusCode) return;

            var result = JsonSerializer.Deserialize<SyncExchangeResponse>(await response.Content.ReadAsStringAsync(), new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
            if (result == null) return;

            if (result.ToPush != null && result.ToPush.Count > 0)
            {
                foreach (var item in result.ToPush)
                {
                    var local = await db.Nodes.FirstOrDefaultAsync(n => n.Uuid == item.Node.Uuid);
                    if (local == null)
                    {
                        db.Nodes.Add(item.Node);
                        db.AclEntries.AddRange(item.AclEntries);
                        db.NodeKeys.AddRange(item.NodeKeys);
                    }
                    else if (local.UpdatedAt < item.Node.UpdatedAt)
                    {
                        local.Name = item.Node.Name;
                        local.Type = item.Node.Type;
                        local.Content = item.Node.Content;
                        local.OwnerUsername = item.Node.OwnerUsername;
                        local.UpdatedAt = item.Node.UpdatedAt;
                        local.IsDeleted = item.Node.IsDeleted;

                        var existingAcls = await db.AclEntries.Where(a => a.NodeUuid == item.Node.Uuid).ToListAsync();
                        db.AclEntries.RemoveRange(existingAcls);
                        db.AclEntries.AddRange(item.AclEntries);

                        var existingKeys = await db.NodeKeys.Where(k => k.NodeUuid == item.Node.Uuid).ToListAsync();
                        db.NodeKeys.RemoveRange(existingKeys);
                        db.NodeKeys.AddRange(item.NodeKeys);
                    }
                }
                await db.SaveChangesAsync();
            }

            if (result.ToRequest != null && result.ToRequest.Count > 0)
            {
                var pushItems = new List<SyncPushItem>();
                foreach (var uuid in result.ToRequest)
                {
                    var node = await db.Nodes.FirstOrDefaultAsync(n => n.Uuid == uuid);
                    if (node != null)
                    {
                        var acls = await db.AclEntries.Where(a => a.NodeUuid == uuid).ToListAsync();
                        var keys = await db.NodeKeys.Where(k => k.NodeUuid == uuid).ToListAsync();
                        pushItems.Add(new SyncPushItem(node, acls, keys));
                    }
                }

                if (pushItems.Count > 0)
                {
                    var pushRequest = new HttpRequestMessage(HttpMethod.Post, $"{peerUrl.TrimEnd('/')}/api/sync/push");
                    pushRequest.Content = new StringContent(JsonSerializer.Serialize(pushItems), Encoding.UTF8, "application/json");
                    pushRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
                    await httpClient.SendAsync(pushRequest);
                }
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error synchronizing with peer {peerUrl}: {ex.Message}");
        }
    }

    public static async Task BootstrapFromPeerServerAsync(string peerUrl, string serverGuid, string sessionToken, ServerDbContext db, HttpClient httpClient)
    {
        try
        {
            Console.WriteLine($"Performing full database pulling bootstrap from {peerUrl}...");

            var request = new HttpRequestMessage(HttpMethod.Post, $"{peerUrl.TrimEnd('/')}/api/sync/pull-database");
            request.Headers.Add("X-Server-Id", serverGuid);
            request.Headers.Add("X-Server-Token", sessionToken);

            var response = await httpClient.SendAsync(request);
            if (!response.IsSuccessStatusCode)
            {
                Console.WriteLine($"Full database pull failed: {response.StatusCode}");
                return;
            }

            var dump = JsonSerializer.Deserialize<DatabaseDumpPayload>(await response.Content.ReadAsStringAsync(), new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
            if (dump == null) return;

            db.Users.RemoveRange(await db.Users.ToListAsync());
            db.Nodes.RemoveRange(await db.Nodes.ToListAsync());
            db.NodeKeys.RemoveRange(await db.NodeKeys.ToListAsync());
            db.PendingShares.RemoveRange(await db.PendingShares.ToListAsync());
            db.AclEntries.RemoveRange(await db.AclEntries.ToListAsync());
            await db.SaveChangesAsync();

            db.Users.AddRange(dump.Users);
            db.Nodes.AddRange(dump.Nodes);
            db.NodeKeys.AddRange(dump.NodeKeys);
            db.PendingShares.AddRange(dump.PendingShares);
            db.AclEntries.AddRange(dump.AclEntries);
            await db.SaveChangesAsync();

            Console.WriteLine("Full database pull bootstrap completed successfully!");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error during full database pulling bootstrap: {ex.Message}");
        }
    }
}

public record RegisterPeerRequest(
    string PeerGuid,
    string PeerUrl,
    string EncryptedSecret,
    string PublicKey,
    string Signature
);

public record BroadcastTarget(string Guid, string Url, string Token, bool IsUpstream);

public record HandshakeResponse(string SessionToken, string ServerGuid);

public record DatabaseDumpPayload(
    List<User> Users,
    List<Node> Nodes,
    List<NodeKey> NodeKeys,
    List<PendingShare> PendingShares,
    List<AclEntry> AclEntries
);

public record SyncNodeState(string Uuid, DateTime UpdatedAt, bool IsDeleted);
public record SyncExchangeRequest(List<SyncNodeState> Items);
public record SyncPushItem(Node Node, List<AclEntry> AclEntries, List<NodeKey> NodeKeys, string? BroadcastId = null, List<PendingShare>? PendingShares = null);
public record SyncExchangeResponse(List<SyncPushItem> ToPush, List<string> ToRequest);

public record PingResponse(string PublicKey, string Timestamp, string Token);

public record UserBroadcastPayload(User User, string? BroadcastId = null);