updated version for scaling the paint only not whole layout:

// MIT License
//
// Copyright (c) 2025 Simon Lightfoot
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//

import 'dart:collection';

import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/scheduler.dart' show SchedulerBinding, SchedulerPhase;

/// Idea: https://x.com/aloisdeniel/status/1942685270102409666

const debugTestClippingInset = 50.0;

void main() {
  runApp(const App());
}

class App extends StatefulWidget {
  const App({super.key});

  @override
  State<App> createState() => _AppState();
}

class _AppState extends State<App> with SingleTickerProviderStateMixin {
  late StackCanvasController _controller;

  @override
  void initState() {
    super.initState();
    _controller = StackCanvasController();
  }

  @override
  void dispose() {
    _controller.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      debugShowCheckedModeBanner: false,
      home: Material(
        child: DefaultTextStyle.merge(
          style: TextStyle(fontSize: 20.0, fontWeight: FontWeight.w500),
          child: StackCanvas(
            controller: _controller,
            children: [
              StackItem(
                rect: Rect.fromLTWH(100, -20, 200, 150),
                builder:
                    (BuildContext context) =>
                        DemoItem(color: Colors.red, label: 'Child 1'),
              ),
              StackItem(
                rect: Rect.fromLTWH(-50, 100, 200, 150),
                builder:
                    (BuildContext context) =>
                        DemoItem(color: Colors.blue, label: 'Child 2'),
              ),
              StackItem(
                rect: Rect.fromLTWH(200, 250, 200, 150),
                builder:
                    (BuildContext context) =>
                        DemoItem(color: Colors.green, label: 'Child 3'),
              ),
              StackItem(
                rect: Rect.fromLTWH(500, 25, 200, 150),
                builder:
                    (BuildContext context) =>
                        DemoItem(color: Colors.teal, label: 'Child 4'),
              ),
            ],
          ),
        ),
      ),
    );
  }
}

class DemoItem extends StatelessWidget {
  const DemoItem({super.key, required this.color, required this.label});

  final Color color;
  final String label;

  @override
  Widget build(BuildContext context) {
    return DecoratedBox(
      decoration: BoxDecoration(
        color: color,
        borderRadius: BorderRadius.circular(16.0),
      ),
      child: Center(child: Text(label)),
    );
  }
}

class StackItem extends StatelessWidget {
  const StackItem({super.key, required this.rect, required this.builder});

  final Rect rect;
  final WidgetBuilder builder;

  @override
  Widget build(BuildContext context) {
    return Positioned.fromRect(rect: rect, child: Builder(builder: builder));
  }
}

class StackCanvasController extends ChangeNotifier {
  StackCanvasController({Offset initialPosition = Offset.zero, double initialScale = 1.0})
      : _origin = initialPosition,
        _scale = initialScale;

  Offset _origin;

  Offset get origin => _origin;

  set origin(Offset value) {
    if (_origin != value) {
      _origin = value;
      notifyListeners();
    }
  }

  double _scale;

  double get scale => _scale;

  set scale(double value) {
    if (_scale != value) {
      _scale = value.clamp(0.1, 10.0); // Limit scale to reasonable range
      notifyListeners();
    }
  }
}

class StackCanvas extends StatefulWidget {
  const StackCanvas({
    super.key,
    required this.controller,
    required this.children,
  });

  final StackCanvasController controller;
  final List<StackItem> children;

  @override
  State<StackCanvas> createState() => _StackCanvasState();
}

class _StackCanvasState extends State<StackCanvas> {
  double _previousScaleFactor = 1.0;

  @override
  Widget build(BuildContext context) {
    return GestureDetector(
      behavior: HitTestBehavior.opaque,
      onScaleStart: (details) {
        _previousScaleFactor = 1.0;
      },
      onScaleUpdate: (details) {
        // Handle pan (always, even during scale)
        widget.controller.origin -= details.focalPointDelta / widget.controller.scale;

        // Handle zoom
        final oldScale = widget.controller.scale;
        final currentScaleFactor = details.scale;
        final deltaScale = currentScaleFactor / _previousScaleFactor;
        final newScale = oldScale * deltaScale;
        if (newScale != oldScale) {
          // Adjust origin to zoom around local focal point
          final deltaOrigin = details.localFocalPoint * (1 / oldScale - 1 / newScale);
          widget.controller.origin += deltaOrigin;
          widget.controller.scale = newScale;
        }
        _previousScaleFactor = currentScaleFactor;
      },
      child: StackCanvasLayout(controller: widget.controller, children: widget.children),
    );
  }
}

class StackCanvasLayout extends RenderObjectWidget {
  const StackCanvasLayout({
    super.key,
    required this.controller,
    required this.children,
  });

  final StackCanvasController controller;
  final List<StackItem> children;

  @override
  RenderObjectElement createElement() => StackCanvasElement(this);

  @protected
  bool updateShouldRebuild(covariant StackCanvasLayout oldWidget) => true;

  @override
  RenderObject createRenderObject(BuildContext context) {
    return RenderStackCanvas(controller: controller);
  }

  @override
  void updateRenderObject(
    BuildContext context,
    covariant RenderStackCanvas renderObject,
  ) {
    renderObject.controller = controller;
  }
}

class StackCanvasElement extends RenderObjectElement {
  StackCanvasElement(StackCanvasLayout super.widget);

  @override
  RenderStackCanvas get renderObject => super.renderObject as RenderStackCanvas;

  @override
  StackCanvasLayout get widget => super.widget as StackCanvasLayout;

  @override
  BuildScope get buildScope => _buildScope;

  late final _buildScope = BuildScope(scheduleRebuild: _scheduleRebuild);

  bool _deferredCallbackScheduled = false;

  void _scheduleRebuild() {
    if (_deferredCallbackScheduled) {
      return;
    }

    final bool deferMarkNeedsLayout = switch (SchedulerBinding.instance.schedulerPhase) {
      SchedulerPhase.idle || SchedulerPhase.postFrameCallbacks => true,
      SchedulerPhase.transientCallbacks ||
      SchedulerPhase.midFrameMicrotasks ||
      SchedulerPhase.persistentCallbacks => false,
    };
    if (!deferMarkNeedsLayout) {
      renderObject.scheduleLayoutCallback();
      return;
    }
    _deferredCallbackScheduled = true;
    SchedulerBinding.instance.scheduleFrameCallback(_frameCallback);
  }

  void _frameCallback(Duration timestamp) {
    _deferredCallbackScheduled = false;
    if (mounted) {
      renderObject.scheduleLayoutCallback();
    }
  }

  var _children = <Element>[];

  /// The current list of children of this element.
  ///
  /// This list is filtered to hide elements that have been forgotten (using
  /// [forgetChild]).
  Iterable<Element> get children =>
      _children.where((Element child) => !_forgottenChildren.contains(child));

  // We keep a set of forgotten children to avoid O(n^2) work walking _children
  // repeatedly to remove children.
  final Set<Element> _forgottenChildren = HashSet<Element>();

  @override
  void visitChildren(ElementVisitor visitor) {
    for (final Element child in _children) {
      if (!_forgottenChildren.contains(child)) {
        visitor(child);
      }
    }
  }

  @override
  void mount(Element? parent, Object? newSlot) {
    super.mount(parent, newSlot);
    renderObject.elementCallback = elementCallback;
  }

  @override
  void update(StackCanvasLayout newWidget) {
    super.update(newWidget);
    renderObject.elementCallback = elementCallback;
    if (newWidget.updateShouldRebuild(widget)) {
      _needsBuild = true;
      renderObject.scheduleLayoutCallback();
    }
  }

  @override
  void markNeedsBuild() {
    renderObject.scheduleLayoutCallback();
    _needsBuild = true;
  }

  @override
  void performRebuild() {
    renderObject.scheduleLayoutCallback();
    _needsBuild = true;
    super.performRebuild();
  }

  @override
  void unmount() {
    renderObject.elementCallback = null;
    super.unmount();
  }

  Rect? _currentViewport;
  bool _needsBuild = true;

  void elementCallback(Rect viewport) {
    if (_needsBuild || _currentViewport != viewport) {
      owner!.buildScope(this, () {
        try {
          // Loop over all widget.children and build the ones that are visible
          final newChildren = widget.children.where((child) {
            return child.rect.overlaps(viewport);
          }).toList();
          _children = updateChildren(
            _children,
            newChildren,
            forgottenChildren: _forgottenChildren,
          );
          _forgottenChildren.clear();
        } finally {
          _needsBuild = false;
          _currentViewport = viewport;
        }
      });
    }
  }

  @override
  void forgetChild(Element child) {
    _forgottenChildren.add(child);
    super.forgetChild(child);
  }

  @override
  void insertRenderObjectChild(RenderBox child, IndexedSlot<Element?> slot) {
    renderObject.insert(child, after: slot.value?.renderObject as RenderBox?);
  }

  @override
  void moveRenderObjectChild(
    RenderBox child,
    IndexedSlot<Element?> oldSlot,
    IndexedSlot<Element?> newSlot,
  ) {
    renderObject.move(child, after: newSlot.value?.renderObject as RenderBox?);
  }

  @override
  void removeRenderObjectChild(RenderBox child, Object? slot) {
    renderObject.remove(child);
  }
}

class RenderStackCanvas extends RenderBox
    with
        ContainerRenderObjectMixin<RenderBox, StackParentData>,
        RenderBoxContainerDefaultsMixin<RenderBox, StackParentData>,
        RenderObjectWithLayoutCallbackMixin {
  RenderStackCanvas({required StackCanvasController controller})
      : _controller = controller;

  StackCanvasController _controller;

  StackCanvasController get controller => _controller;

  set controller(StackCanvasController value) {
    if (_controller != value) {
      if (attached) {
        _controller.removeListener(_onControllerChanged);
        value.addListener(_onControllerChanged);
      }
      _controller = value;
      _onControllerChanged();
    }
  }

  void Function(Rect viewport)? _elementCallback;

  set elementCallback(void Function(Rect viewport)? value) {
    if (_elementCallback != value) {
      _elementCallback = value;
      if (_elementCallback != null) {
        scheduleLayoutCallback();
      }
    }
  }

  @override
  void attach(PipelineOwner owner) {
    super.attach(owner);
    controller.addListener(_onControllerChanged);
  }

  @override
  void detach() {
    controller.removeListener(_onControllerChanged);
    super.detach();
  }

  void _onControllerChanged() {
    scheduleLayoutCallback();
  }

  @override
  void setupParentData(RenderBox child) {
    if (child.parentData is! StackParentData) {
      child.parentData = StackParentData();
    }
  }

  @override
  void layoutCallback() {
    final scale = controller.scale;
    final buffer = debugTestClippingInset / scale;
    final worldWidth = constraints.maxWidth / scale;
    final worldHeight = constraints.maxHeight / scale;
    final viewport = Rect.fromLTWH(
      controller.origin.dx,
      controller.origin.dy,
      worldWidth,
      worldHeight,
    ).deflate(buffer);
    if (_elementCallback != null) {
      _elementCallback!(viewport);
    }
  }

  @override
  void performLayout() {
    runLayoutCallback();

    final children = getChildrenAsList();
    for (final child in children) {
      final parentData = child.parentData as StackParentData;
      final childConstraints = BoxConstraints.tightFor(
        width: parentData.width,
        height: parentData.height,
      );
      child.layout(childConstraints);
      parentData.offset = Offset(
        parentData.left ?? 0,
        parentData.top ?? 0,
      );
    }

    size = constraints.biggest;
  }

  @override
  bool hitTestChildren(BoxHitTestResult result, {required Offset position}) {
    final scale = controller.scale;
    if (scale == 0) return false;
    final worldPosition = controller.origin + position / scale;
    return defaultHitTestChildren(result, position: worldPosition);
  }

  @override
  void paint(PaintingContext context, Offset offset) {
    final scale = controller.scale;
    context.canvas.save();
    context.canvas.translate(
      offset.dx - controller.origin.dx * scale,
      offset.dy - controller.origin.dy * scale,
    );
    context.canvas.scale(scale, scale);
    defaultPaint(context, Offset.zero);
    context.canvas.restore();

    if (debugPaintSizeEnabled) {
      context.canvas.drawRect(
        (offset & size).deflate(debugTestClippingInset),
        Paint()
          ..style = PaintingStyle.stroke
          ..strokeWidth = 3.0
          ..color = Color(0xFFFF00FF),
      );
    }
  }
}