using System.Drawing;

internal class ExtendedRect {

    private Rectangle _rect;

    private sealed record SideDistance(int Side, int Distance);

    public long Area {
        get {
            return _rect.Width * _rect.Height;
        }
    }

    #region Factory

    public ExtendedRect(Rectangle rect) {
        _rect = new(rect.Location, rect.Size);
    }

    public ExtendedRect(int x, int y, int width, int height) {
        _rect = new(x, y, width, height);
    }

    public ExtendedRect(Point location, Size size) {
        _rect = new(location, size);
    }

    #endregion Factory

    public Size Size => _rect.Size;
    public Point Location => _rect.Location;
    public bool IsEmpty => _rect.IsEmpty;


    /// <summary>
    /// Given a two dimensional space point it tries to find which side of the embedded
    /// rectangle is the nearest to the point itself.
    /// </summary>
    /// <param name="point">A <see cref="Point"/></param>
    /// <param name="distanceThreshold">The maximum distance value allowed to consider a side as "near" the provided point</param>
    /// <returns>-1 when no side found matching the criteria or a value in [0,3]</returns>
    /// <remarks>
    /// - Whether the point is contained in the rectangle is not relevant
    /// - A rectangle with zero area will always return -1 (there are no sides)
    /// - The value returned when != -1 is the index of the matching side in clockwise order starting from Left
    /// </remarks>
    public int FindNearestSideLoop(Point point, int distanceThreshold = 25) {
        int ret = -1;
        if (Area == 0 || (point.X < 0 || point.Y < 0)) { return ret; }

        for (int i = 0; i < 4; ++i) {
            int distance = i switch {
                0 => Math.Abs(_rect.Left - point.X),
                1 => Math.Abs(_rect.Top - point.Y),
                2 => Math.Abs(_rect.Right - point.X),
                3 => Math.Abs(_rect.Bottom - point.Y),
            };

            if (distance < distanceThreshold) {
                ret = i;
                distanceThreshold = distance;
                if (distanceThreshold < 1) {
                    break; // Found a point ON a side.
                }
            }
        }
        return ret;
    }

    public int FindNearestSideLinq(Point point, int distanceThreshold = 25) {

        if (Area == 0 || (point.X < 0 || point.Y < 0)) { return -1; }
        var distances = new List<SideDistance> {
            new(0, Math.Abs(_rect.Left - point.X)),
            new(1, Math.Abs(_rect.Top - point.Y)),
            new(2, Math.Abs(_rect.Right - point.X)),
            new(3, Math.Abs(_rect.Bottom - point.Y))
        };

        return distances.Where(d => d.Distance < distanceThreshold).OrderBy(d => d.Distance).FirstOrDefault()?.Side ?? -1;
    }

}

class Program {

    static void Main() {

        var TargetRect = new ExtendedRect(1024, 1024, 8192, 8192);
        var RegionRect = new Rectangle(0, 0, TargetRect.Size.Width + 2048, TargetRect.Size.Height + 2048);

        // Create a list of unique points randomly picked from RegionRect
        var random = new Random();
        var points = new HashSet<Point>();
        while (points.Count < 100_000) {
            _ = points.Add(new Point(random.Next(RegionRect.Left, RegionRect.Right), random.Next(RegionRect.Top, RegionRect.Bottom)));
        }

        Console.WriteLine($"Points To Test: {points.Count}");

        int matchings = 0;
        int misses = 0;
        var foundSides = new List<int>();

        var watch = System.Diagnostics.Stopwatch.StartNew();
        foreach (var point in points) {
            foundSides.Add(TargetRect.FindNearestSideLoop(point));
        }
        watch.Stop();
        matchings = foundSides.Count(f => f != -1);
        misses = foundSides.Count(f => f == -1);
        Console.Write($"Loop: {watch.ElapsedMilliseconds}ms matches:{matchings} misses:{misses}\n");

        foundSides.Clear();
        watch.Restart();
        foreach (var point in points) {
            foundSides.Add(TargetRect.FindNearestSideLinq(point));
        }
        watch.Stop();
        matchings = foundSides.Count(f => f != -1);
        misses = foundSides.Count(f => f == -1);
        Console.Write($"Linq: {watch.ElapsedMilliseconds}ms matches:{matchings} misses:{misses}\n");

    }
}

Results :

Points To Test: 100000
Loop: 7ms matches:1924 misses:98076
Linq: 42ms matches:1924 misses:98076