Leave out deep subtrees instead of overflowing the stack

CHANGELOG.md

@@ -5,6 +5,7 @@ Released on Wednesday, November 11 2026.
 - Improved the performance of nested tables and inline-blocks, which were laid out twice as often with every level (#54) @michaelthyregod
 - Fixed adjacent text nodes (e.g. inserted by a script or a framework like Blazor) each starting a line of their own in a block, table cell or flex item: they are laid out as one text, like the single node the parser makes of the same text
 - Fixed the intrinsic width of a row flex container counting each of adjacent text nodes as an item of its own (with a gap between them)
+- Improved deeply nested pages: what is nested too deep for the stack is left out instead of ending the process with a stack overflow (#55) @michaelthyregod
 
 # 0.11.1
 

src/AngleSharp.Renderer.Tests/DeepNestingTests.cs

@@ -0,0 +1,70 @@
+namespace AngleSharp.Renderer.Tests;
+
+using System.Text;
+using System.Threading;
+using AngleSharp;
+using AngleSharp.Css;
+
+/// <summary>
+/// Deeply nested documents: layout recurses per nesting level, so a deep enough tree must leave
+/// its innermost content out rather than overflow the stack (which ends the whole process).
+/// </summary>
+public sealed class DeepNestingTests
+{
+    private static readonly DefaultRenderDevice Device = new() { ViewPortWidth = 300, ViewPortHeight = 200, FontSize = 16 };
+
+    // The HTML parser keeps at most 511 levels below the body, so this is the deepest markup can get.
+    private const int ParserDepthLimit = 511;
+
+    // A thread-pool thread or a desktop app's thread commonly has 1 MB of stack.
+    private const int StackSize = 1024 * 1024;
+
+    [Theory]
+    [InlineData("<div>", "</div>")]
+    [InlineData("<span>x", "</span>")]
+    [InlineData("<div style='display: flex'>", "</div>")]
+    [InlineData("<div style='display: grid'>", "</div>")]
+    [InlineData("<div style='display: inline-block'>", "</div>")]
+    [InlineData("<div style='float: left'>", "</div>")]
+    [InlineData("<table><tr><td>", "</td></tr></table>")]
+    public async Task DeepestParsableNesting_RendersWithoutOverflowingTheStack(string open, string close)
+    {
+        var markup = new StringBuilder("<!doctype html><html><body>");
+
+        for (var i = 0; i < ParserDepthLimit; i++)
+        {
+            markup.Append(open);
+        }
+
+        markup.Append("leaf");
+
+        for (var i = 0; i < ParserDepthLimit; i++)
+        {
+            markup.Append(close);
+        }
+
+        markup.Append("</body></html>");
+        var context = BrowsingContext.New(Configuration.Default.WithCss().WithRenderDevice(Device));
+        var document = await context.OpenAsync(request => request.Content(markup.ToString()));
+        byte[]? png = null;
+        Exception? error = null;
+        var thread = new Thread(() =>
+        {
+            try
+            {
+                png = new HtmlRenderer().RenderToPng(document, Device).Data;
+            }
+            catch (Exception ex)
+            {
+                error = ex;
+            }
+        }, StackSize);
+
+        thread.Start();
+        thread.Join();
+
+        Assert.Null(error);
+        Assert.NotNull(png);
+        Assert.NotEmpty(png);
+    }
+}

src/AngleSharp.Renderer/HtmlRenderer.Inline.cs

@@ -545,7 +545,8 @@ public void Collect(IRenderNode node, RenderTextStyle style, int styleId, Inline
 
         private void CollectElement(ElementRenderNode element, RenderTextStyle parentStyle, InlineBox? parentBox)
         {
-            if (!GeneratesBoxes(element))
+            // a deep subtree is left out once the stack runs low (see LayoutNode)
+            if (!GeneratesBoxes(element) || !RuntimeHelpers.TryEnsureSufficientExecutionStack())
             {
                 return;
             }
@@ -3096,6 +3097,12 @@ private static IntrinsicWidths ComputeElementContribution(ElementRenderNode elem
     /// </summary>
     private static IntrinsicWidths ComputeContentIntrinsicWidths(ElementRenderNode element, RenderTextStyle textStyle, float percentageBase, LayoutContext context)
     {
+        // a deep subtree is left out once the stack runs low (see LayoutNode)
+        if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+        {
+            return new IntrinsicWidths(0f, 0f);
+        }
+
         var widths = ComputeElementContentWidths(element, textStyle, percentageBase, context);
 
         if (s_layoutPass.Value is { } pass)

src/AngleSharp.Renderer/HtmlRenderer.cs

@@ -1241,6 +1241,13 @@ private static void LayoutNode(
                 LayoutTextNode(textNode.Ref, containingX, containingWidth, ref cursorY, ref previousBlockMarginBottom, ref suppressNextBlockTopMargin, floats, ref textIndentConsumed, textStyle, context, displayList, maxY);
                 return;
             case ElementRenderNode element:
+                // Every nesting level takes a large stack frame; when the stack runs low the rest of
+                // the subtree is left out instead of ending the process with a stack overflow.
+                if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+                {
+                    return;
+                }
+
                 LayoutStickyAwareElement(element, containingX, containingY, containingWidth, ref cursorY, ref previousBlockMarginBottom, ref suppressNextBlockTopMargin, floats, ref textIndentConsumed, textStyle, context, displayList, maxY, isFlexItem, isRowDirection, flexMainSize, flexCrossSize, containingHeight, positionedContainingX, positionedContainingY, positionedContainingWidth, positionedContainingHeight, forcedBorderBoxWidth, isAtomicInline, forcedBorderBoxHeight, contentOffsetY, mainAxisMarginsPlaced, crossAxisMarginsPlaced);
                 return;
             default:
@@ -8308,6 +8315,13 @@ private static List<AuthoredRule> GetAuthoredRules(IDocument document, string pr
 
     private static bool TryGetFirstCollapsibleChildTopMargin(ElementRenderNode node, float containingWidth, LayoutContext context, out float marginTop)
     {
+        // a deep subtree is left out once the stack runs low (see LayoutNode)
+        if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+        {
+            marginTop = 0f;
+            return false;
+        }
+
         var throughPositive = 0f;
         var throughNegative = 0f;
         var hasThrough = false;
@@ -8390,6 +8404,12 @@ private static bool TryGetFirstCollapsibleChildTopMargin(ElementRenderNode node,
     /// </summary>
     private static bool IsSelfCollapsing(ElementRenderNode element, LayoutContext context)
     {
+        // a deep subtree is left out once the stack runs low (see LayoutNode)
+        if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+        {
+            return false;
+        }
+
         var styleMap = GetOrCreateStyleMap(context, element.Ref, element.ComputedStyle);
 
         if (!IsFlowBlockDisplay(styleMap) || IsReplacedElement(element.Ref) || ResolveFormControlKind(element) != FormControlKind.None)