From e73aa472effa8c255e992043f98086d6dc49460f Mon Sep 17 00:00:00 2001 From: Justin Haygood Date: Mon, 13 Jul 2026 22:07:47 -0400 Subject: [PATCH 1/3] Use HtmlKit tokenizer for HTML5 parsing Replaces the hand-rolled HTML scanner with HtmlKit tokenization and updates DOM parsing to better follow HTML5 behavior, including omitted end-tag handling (`p`/`td`/`tr`), safer comment/script handling, and `noscript` re-parsing. Adds anonymous table box correction logic and related DOM/CSS utilities, migrates text storage from `SubString` to `string` (removing `SubString.cs`), and includes a new demo sample showcasing malformed-HTML rendering improvements. --- .../Common/HtmlRenderer.Demo.Common.csproj | 1 + .../Demo/Common/Samples/15.HTML5 Parsing.htm | 124 ++++++++ Source/HtmlRenderer/Core/Dom/CssBox.cs | 26 +- .../HtmlRenderer/Core/Dom/CssLayoutEngine.cs | 2 +- Source/HtmlRenderer/Core/Parse/DomParser.cs | 202 ++++++++++--- Source/HtmlRenderer/Core/Parse/HtmlParser.cs | 279 +++++++----------- Source/HtmlRenderer/Core/Utils/CommonUtils.cs | 11 + Source/HtmlRenderer/Core/Utils/DomUtils.cs | 46 ++- .../HtmlRenderer/Core/Utils/HtmlConstants.cs | 2 +- Source/HtmlRenderer/Core/Utils/HtmlUtils.cs | 78 +++++ Source/HtmlRenderer/Core/Utils/SubString.cs | 187 ------------ Source/HtmlRenderer/HtmlRenderer.csproj | 4 + 12 files changed, 546 insertions(+), 416 deletions(-) create mode 100644 Source/Demo/Common/Samples/15.HTML5 Parsing.htm delete mode 100644 Source/HtmlRenderer/Core/Utils/SubString.cs diff --git a/Source/Demo/Common/HtmlRenderer.Demo.Common.csproj b/Source/Demo/Common/HtmlRenderer.Demo.Common.csproj index b2b829640..d6a6a96d0 100644 --- a/Source/Demo/Common/HtmlRenderer.Demo.Common.csproj +++ b/Source/Demo/Common/HtmlRenderer.Demo.Common.csproj @@ -87,6 +87,7 @@ + diff --git a/Source/Demo/Common/Samples/15.HTML5 Parsing.htm b/Source/Demo/Common/Samples/15.HTML5 Parsing.htm new file mode 100644 index 000000000..bfa8df0a1 --- /dev/null +++ b/Source/Demo/Common/Samples/15.HTML5 Parsing.htm @@ -0,0 +1,124 @@ + + + HTML5 Parsing + + + +

HTML5 Parsing +

+
+

+ The html is parsed using a real HTML5 tokenizer (from the + HtmlKit library) instead of a hand-rolled + scanner, so the pages below - all of which are intentionally malformed - render + correctly instead of breaking the layout. Every example shows the raw (broken) html + source followed by the actual live rendering of that exact same markup. +

+
+

Optional / omitted end tags +

+

+ Real world html is full of tags whose closing tag was never written. The parser now + follows the + HTML5 end-tag omission rules and automatically closes them at the right + place, instead of nesting everything into one deep, broken tree. +

+
Source:
+ 
<p>First paragraph, never closed
+<p>Second paragraph, never closed
+<p>Third paragraph, never closed
+
Live result:
+
+

First paragraph, never closed +

Second paragraph, never closed +

Third paragraph, never closed +

+
+

Tables with missing <tr>/<td> end tags +

+

+ The same omission rules apply to table rows and cells, so a table written without a + single closing </tr> or </td> still comes out as + a proper grid. +

+
Source:
+ 
<table border="1" cellpadding="4">
+<tr><td>A<td>B
+<tr><td>C<td>D
+</table>
+
Live result:
+
+ +
AB +
CD +
+
+
+

Anonymous table boxes +

+

+ Per the CSS 2.1 anonymous + table object rules, a stray <td> with no <tr> + (or even no <table>) around it is automatically wrapped in the missing + row/table boxes, rather than producing a broken or invisible layout. +

+
Source:
+ 
<table border="1"><td>Anonymous row generated around me</td></table>
+
Live result:
+
+
Anonymous row generated around me
+
+
+

Comments containing ">" +

+

+ A real tokenizer knows where a comment actually ends, so a stray > character + inside a comment no longer truncates it early or corrupts the rest of the page. +

+
Source:
+ 
<!-- a comment that contains a > character right here -->
+<p>This paragraph renders normally right after it.</p>
+
Live result:
+
+ +

This paragraph renders normally right after it.

+
+
+

<script> content is not mistaken for markup +

+

+ <script> content is tokenized as raw text (this renderer does not execute + script, so the code itself is not shown), which means < and > + characters used for comparisons inside the script can no longer be misread as the start + of a nested tag and swallow the rest of the document. +

+
Source:
+ 
<script>
+    if (1 < 2 && 3 > 2) { runDemo('<b>this looks like a tag but is just a string</b>'); }
+</script>
+<p>This paragraph still renders correctly right after the script block.</p>
+
Live result:
+
+ +

This paragraph still renders correctly right after the script block.

+
+
+

<noscript> content is parsed as real markup +

+

+ Per the HTML5 tokenizer spec, <noscript> content is normally tokenized + as raw text too. Since this renderer never runs script, its <noscript> + content is specifically re-parsed as nested html, so tags placed inside it render as real + elements instead of showing up as literal text. +

+
Source:
+ 
<noscript><p>This is <b>real, nested markup</b> inside noscript.</p></noscript>
+
Live result:
+
+ +
+
+ + diff --git a/Source/HtmlRenderer/Core/Dom/CssBox.cs b/Source/HtmlRenderer/Core/Dom/CssBox.cs index b2cd3d984..05ae7f539 100644 --- a/Source/HtmlRenderer/Core/Dom/CssBox.cs +++ b/Source/HtmlRenderer/Core/Dom/CssBox.cs @@ -60,7 +60,7 @@ internal class CssBox : CssBoxProperties, IDisposable /// /// the inner text of the box /// - private SubString _text; + private string _text; /// /// Do not use or alter this flag @@ -146,6 +146,14 @@ public bool IsBrElement } } + /// + /// Is the box "Display" is one of the table-row-group/table-header-group/table-footer-group values. + /// + public bool IsTableRowGroupBox + { + get { return Display == CssConstants.TableRowGroup || Display == CssConstants.TableHeaderGroup || Display == CssConstants.TableFooterGroup; } + } + /// /// is the box "Display" is "Inline", is this is an inline box and not block. /// @@ -278,7 +286,7 @@ public bool IsSpaceOrEmpty /// /// Gets or sets the inner text of the box /// - public SubString Text + public string Text { get { return _text; } set @@ -541,7 +549,7 @@ public void ParseToWords() int startIdx = 0; bool preserveSpaces = WhiteSpace == CssConstants.Pre || WhiteSpace == CssConstants.PreWrap; - bool respoctNewline = preserveSpaces || WhiteSpace == CssConstants.PreLine; + bool respoctNewline = preserveSpaces || WhiteSpace == CssConstants.PreLine || IsBrElement; while (startIdx < _text.Length) { while (startIdx < _text.Length && _text[startIdx] == '\r') @@ -795,27 +803,27 @@ private void CreateListItemBox(RGraphics g) if (ListStyleType.Equals(CssConstants.Disc, StringComparison.InvariantCultureIgnoreCase)) { - _listItemBox.Text = new SubString("•"); + _listItemBox.Text = "•"; } else if (ListStyleType.Equals(CssConstants.Circle, StringComparison.InvariantCultureIgnoreCase)) { - _listItemBox.Text = new SubString("o"); + _listItemBox.Text = "o"; } else if (ListStyleType.Equals(CssConstants.Square, StringComparison.InvariantCultureIgnoreCase)) { - _listItemBox.Text = new SubString("♠"); + _listItemBox.Text = "♠"; } else if (ListStyleType.Equals(CssConstants.Decimal, StringComparison.InvariantCultureIgnoreCase)) { - _listItemBox.Text = new SubString(GetIndexForList().ToString(CultureInfo.InvariantCulture) + "."); + _listItemBox.Text = GetIndexForList().ToString(CultureInfo.InvariantCulture) + "."; } else if (ListStyleType.Equals(CssConstants.DecimalLeadingZero, StringComparison.InvariantCultureIgnoreCase)) { - _listItemBox.Text = new SubString(GetIndexForList().ToString("00", CultureInfo.InvariantCulture) + "."); + _listItemBox.Text = GetIndexForList().ToString("00", CultureInfo.InvariantCulture) + "."; } else { - _listItemBox.Text = new SubString(CommonUtils.ConvertToAlphaNumber(GetIndexForList(), ListStyleType) + "."); + _listItemBox.Text = CommonUtils.ConvertToAlphaNumber(GetIndexForList(), ListStyleType) + "."; } _listItemBox.ParseToWords(); diff --git a/Source/HtmlRenderer/Core/Dom/CssLayoutEngine.cs b/Source/HtmlRenderer/Core/Dom/CssLayoutEngine.cs index a24f9e708..e41de241f 100644 --- a/Source/HtmlRenderer/Core/Dom/CssLayoutEngine.cs +++ b/Source/HtmlRenderer/Core/Dom/CssLayoutEngine.cs @@ -344,7 +344,7 @@ private static void FlowBox(RGraphics g, CssBox blockbox, CssBox box, double lim } // handle box that is only a whitespace - if (box.Text != null && box.Text.IsWhitespace() && !box.IsImage && box.IsInline && box.Boxes.Count == 0 && box.Words.Count == 0) + if (box.Text != null && CommonUtils.IsNonEmptyWhitespace(box.Text) && !box.IsImage && box.IsInline && box.Boxes.Count == 0 && box.Words.Count == 0) { curx += box.ActualWordSpacing; } diff --git a/Source/HtmlRenderer/Core/Parse/DomParser.cs b/Source/HtmlRenderer/Core/Parse/DomParser.cs index d841485f0..1556aa783 100644 --- a/Source/HtmlRenderer/Core/Parse/DomParser.cs +++ b/Source/HtmlRenderer/Core/Parse/DomParser.cs @@ -12,6 +12,7 @@ using System; using System.Globalization; +using System.Linq; using TheArtOfDev.HtmlRenderer.Adapters.Entities; using TheArtOfDev.HtmlRenderer.Core.Dom; using TheArtOfDev.HtmlRenderer.Core.Entities; @@ -70,14 +71,15 @@ public CssBox GenerateCssTree(string html, HtmlContainerInt htmlContainer, ref C CorrectImgBoxes(root); - bool followingBlock = true; - CorrectLineBreaksBlocks(root, ref followingBlock); + CorrectLineBreaksBlocks(root); CorrectInlineBoxesParent(root); CorrectBlockInsideInline(root); CorrectInlineBoxesParent(root); + + CorrectAnonymousTables(root); } return root; } @@ -117,7 +119,7 @@ private void CascadeParseStyles(CssBox box, HtmlContainerInt htmlContainer, ref { CloneCssData(ref cssData, ref cssDataChanged); foreach (var child in box.Boxes) - _cssParser.ParseStyleSheet(cssData, child.Text.CutSubstring()); + _cssParser.ParseStyleSheet(cssData, child.Text); } } @@ -576,7 +578,10 @@ private static void CorrectTextBoxes(CssBox box) if (childBox.Text != null) { // is the box has text - var keepBox = !childBox.Text.IsEmptyOrWhitespace(); + var keepBox = !string.IsNullOrWhiteSpace(childBox.Text); + + // is the box a "br" element (its Text is a forced "\n" line break, not real content) + keepBox = keepBox || childBox.IsBrElement; // is the box is pre-formatted keepBox = keepBox || childBox.WhiteSpace == CssConstants.Pre || childBox.WhiteSpace == CssConstants.PreWrap; @@ -633,51 +638,30 @@ private static void CorrectImgBoxes(CssBox box) } /// - /// Correct the DOM tree recursively by replacing "br" html boxes with anonymous blocks that respect br spec.
- /// If the "br" tag is after inline box then the anon block will have zero height only acting as newline, - /// but if it is after block box then it will have min-height of the font size so it will create empty line. + /// Correct the DOM tree recursively by turning "br" html boxes that should force a line break + /// into a "\n" text word on the box itself, reusing the same forced-newline mechanism used for + /// "white-space: pre/pre-line" content. /// /// the current box to correct its sub-tree - /// used to know if the br is following a box so it should create an empty line or not so it only - /// move to a new line - private static void CorrectLineBreaksBlocks(CssBox box, ref bool followingBlock) + private static void CorrectLineBreaksBlocks(CssBox box) { - followingBlock = followingBlock || box.IsBlock; foreach (var childBox in box.Boxes) { - CorrectLineBreaksBlocks(childBox, ref followingBlock); - followingBlock = childBox.Words.Count == 0 && (followingBlock || childBox.IsBlock); + CorrectLineBreaksBlocks(childBox); } - int lastBr = -1; - CssBox brBox; - do - { - brBox = null; - for (int i = 0; i < box.Boxes.Count && brBox == null; i++) - { - if (i > lastBr && box.Boxes[i].IsBrElement) - { - brBox = box.Boxes[i]; - lastBr = i; - } - else if (box.Boxes[i].Words.Count > 0) - { - followingBlock = false; - } - else if (box.Boxes[i].IsBlock) - { - followingBlock = true; - } - } + if (!box.IsBrElement) return; - if (brBox != null) + var previousSibling = DomUtils.GetPreviousSibling(box); + if (previousSibling == null || previousSibling.IsBlock) + { + var nextSibling = DomUtils.GetFollowingSiblings(box, b => b.IsInline && !b.IsBrElement, true).FirstOrDefault(); + if (nextSibling == null) { - brBox.Display = CssConstants.Block; - if (followingBlock) - brBox.Height = ".95em"; // TODO:a check the height to min-height when it is supported + box.Text = "\n"; + box.ParseToWords(); } - } while (brBox != null); + } } /// @@ -895,6 +879,146 @@ private static bool ContainsVariantBoxes(CssBox box) return hasBlock && hasInline; } + /// + /// Corrects the missing elements in tables per https://www.w3.org/TR/CSS2/tables.html#anonymous-boxes + /// + /// the current box to correct its sub-tree + private static void CorrectAnonymousTables(CssBox box) + { + // 1. Remove irrelevant boxes + CorrectAnonymousTablesRemoveIrrelevantBoxes(box); + + foreach (var childBox in box.Boxes.ToArray()) + { + CorrectAnonymousTablesRemoveIrrelevantBoxes(childBox); + } + + // 2. Generate missing child wrappers + CorrectAnonymousTablesGenerateMissingChildWrappers(box); + + foreach (var childBox in box.Boxes.ToArray()) + { + CorrectAnonymousTablesGenerateMissingChildWrappers(childBox); + } + + // 3. Generate missing parents + CorrectAnonymousTablesGenerateMissingParents(box); + + foreach (var childBox in box.Boxes.ToArray()) + { + CorrectAnonymousTablesGenerateMissingParents(childBox); + } + + foreach (var childBox in box.Boxes.ToArray()) + { + CorrectAnonymousTables(childBox); + } + } + + private static void CorrectAnonymousTablesRemoveIrrelevantBoxes(CssBox box) + { + // 1.1 All child boxes of a 'table-column' parent are treated as if they had 'display: none' + if (box.Display == CssConstants.TableColumn) + { + foreach (var childBox in box.Boxes) + { + childBox.Display = CssConstants.None; + } + } + + // 1.2 If a child of a 'table-column-group' parent is not a 'table-column' box, it is treated as if it had 'display: none'. + if (box.ParentBox != null && box.ParentBox.Display == CssConstants.TableColumnGroup && box.Display != CssConstants.TableColumn) + { + box.Display = CssConstants.None; + } + } + + private static void CorrectAnonymousTablesGenerateMissingChildWrappers(CssBox box) + { + // 2.1 If a child of a 'table'/'inline-table' box is not a proper table child, generate an anonymous 'table-row' box around it + // and all consecutive siblings that are not proper table children. + if (box.ParentBox != null && box.ParentBox.Display == CssConstants.Table) + { + if (!DomUtils.IsProperTableChild(box)) + { + var tableRowBox = new CssBox(box.ParentBox, null); + tableRowBox.Display = CssConstants.TableRow; + box.ParentBox = tableRowBox; + } + } + + // 2.2 If a child of a row group box is not a 'table-row' box, generate an anonymous 'table-row' box around it + // and all consecutive siblings that are not 'table-row' boxes. + if (box.ParentBox != null && box.ParentBox.IsTableRowGroupBox) + { + if (box.Display != CssConstants.TableRow) + { + var tableRowBox = new CssBox(box.ParentBox, null); + tableRowBox.Display = CssConstants.TableRow; + box.ParentBox = tableRowBox; + } + } + + // 2.3 If a child of a 'table-row' box is not a 'table-cell', generate an anonymous 'table-cell' box around it + // and all consecutive siblings that are not 'table-cell' boxes. + if (box.ParentBox != null && box.ParentBox.Display == CssConstants.TableRow) + { + if (box.Display != CssConstants.TableCell) + { + var followingMatchingSiblings = DomUtils.GetFollowingSiblings(box, sibling => sibling.Display == CssConstants.TableCell, true).ToList(); + + var tableCellBox = new CssBox(box.ParentBox, null); + tableCellBox.Display = CssConstants.TableCell; + box.ParentBox = tableCellBox; + + followingMatchingSiblings.ForEach(sib => sib.ParentBox = tableCellBox); + } + } + } + + private static void CorrectAnonymousTablesGenerateMissingParents(CssBox box) + { + // 3.1 For each 'table-cell' box in a sequence of consecutive internal table and 'table-caption' siblings, if its parent + // is not a 'table-row' then generate an anonymous 'table-row' box around it and all consecutive 'table-cell' siblings. + if (box.Display == CssConstants.TableCell) + { + if (box.ParentBox == null || box.ParentBox.Display != CssConstants.TableRow) + { + var followingMatchingSiblings = DomUtils.GetFollowingSiblings(box, sibling => sibling.Display == CssConstants.TableCell, true).ToList(); + + var tableRowBox = new CssBox(box.ParentBox, null); + tableRowBox.Display = CssConstants.TableRow; + box.ParentBox = tableRowBox; + + followingMatchingSiblings.ForEach(sib => sib.ParentBox = tableRowBox); + } + } + + // 3.2 For each proper table child in a sequence of consecutive proper table children, if it is misparented then generate + // an anonymous 'table'/'inline-table' box around it and all consecutive proper-table-child siblings. + if (DomUtils.IsProperTableChild(box)) + { + var isMissingParent = box.ParentBox == null; + var isParentNotTable = box.ParentBox == null || box.ParentBox.Display != CssConstants.Table; + var isParentNotInlineTable = box.ParentBox == null || box.ParentBox.Display != CssConstants.InlineTable; + + var isMisparented = isMissingParent && isParentNotTable && isParentNotInlineTable; + + if (isMisparented) + { + var parentDisplay = (box.ParentBox == null || box.ParentBox.IsBlock) ? CssConstants.Table : CssConstants.InlineTable; + + var followingMatchingSiblings = DomUtils.GetFollowingSiblings(box, DomUtils.IsProperTableChild, true).ToList(); + + var tableBox = new CssBox(box.ParentBox, null); + tableBox.Display = parentDisplay; + box.ParentBox = tableBox; + + followingMatchingSiblings.ForEach(sib => sib.ParentBox = tableBox); + } + } + } + #endregion } } \ No newline at end of file diff --git a/Source/HtmlRenderer/Core/Parse/HtmlParser.cs b/Source/HtmlRenderer/Core/Parse/HtmlParser.cs index 49ee39051..3e85e1997 100644 --- a/Source/HtmlRenderer/Core/Parse/HtmlParser.cs +++ b/Source/HtmlRenderer/Core/Parse/HtmlParser.cs @@ -1,4 +1,4 @@ -// "Therefore those skilled at the unorthodox +// "Therefore those skilled at the unorthodox // are infinite as heaven and earth, // inexhaustible as the great rivers. // When they come to an end, @@ -6,19 +6,23 @@ // like the days and months; // they die and are reborn, // like the four seasons." -// +// // - Sun Tsu, // "The Art of War" using System; using System.Collections.Generic; +using System.IO; +using System.Linq; +using HtmlKit; using TheArtOfDev.HtmlRenderer.Core.Dom; using TheArtOfDev.HtmlRenderer.Core.Utils; +using HtmlUtils = TheArtOfDev.HtmlRenderer.Core.Utils.HtmlUtils; namespace TheArtOfDev.HtmlRenderer.Core.Parse { /// - /// + /// /// internal static class HtmlParser { @@ -26,63 +30,53 @@ internal static class HtmlParser /// Parses the source html to css boxes tree structure. /// /// the html source to parse - public static CssBox ParseDocument(string source) + /// the root box (null for document root) + public static CssBox ParseDocument(string source, CssBox root = null) { - var root = CssBox.CreateBlock(); + if (root == null) + root = CssBox.CreateBlock(); + var curBox = root; - int endIdx = 0; - int startIdx = 0; - while (startIdx >= 0) + using (var sourceReader = new StringReader(source)) { - var tagIdx = source.IndexOf('<', startIdx); - if (tagIdx >= 0 && tagIdx < source.Length) - { - // add the html text as anon css box to the structure - AddTextBox(source, startIdx, tagIdx, ref curBox); + var tokenizer = new HtmlTokenizer(sourceReader); - if (source[tagIdx + 1] == '!') - { - if (source[tagIdx + 2] == '-') - { - // skip the html comment elements () - startIdx = source.IndexOf("-->", tagIdx + 2); - endIdx = startIdx > 0 ? startIdx + 3 : tagIdx + 2; - } - else - { - // skip the html crap elements () - startIdx = source.IndexOf(">", tagIdx + 2); - endIdx = startIdx > 0 ? startIdx + 1 : tagIdx + 2; - } - } - else + HtmlToken token; + while (tokenizer.ReadNextToken(out token)) + { + switch (token.Kind) { - // parse element tag to css box structure - endIdx = ParseHtmlTag(source, tagIdx, ref curBox) + 1; - - if (curBox.HtmlTag != null && curBox.HtmlTag.Name.Equals(HtmlConstants.Style, StringComparison.OrdinalIgnoreCase)) - { - var endIdxS = endIdx; - endIdx = source.IndexOf("", endIdx, StringComparison.OrdinalIgnoreCase); - if (endIdx > -1) - AddTextBox(source, endIdxS, endIdx, ref curBox); - } + case HtmlTokenKind.Tag: + { + var tag = (HtmlTagToken)token; + ParseHtmlTag(tag, ref curBox); + break; + } + case HtmlTokenKind.Data: + { + var text = (HtmlDataToken)token; + + if (curBox.HtmlTag != null && curBox.HtmlTag.Name.Equals(HtmlConstants.NoScript, StringComparison.OrdinalIgnoreCase)) + { + curBox = ParseDocument(text.Data, curBox); + } + else + { + AddTextBox(text, ref curBox); + } + + break; + } + case HtmlTokenKind.CData: + case HtmlTokenKind.Comment: + case HtmlTokenKind.DocType: + case HtmlTokenKind.ScriptData: + break; + default: + throw new ArgumentOutOfRangeException(); } } - startIdx = tagIdx > -1 && endIdx > 0 ? endIdx : -1; - } - - // handle pieces of html without proper structure - if (endIdx > -1 && endIdx < source.Length) - { - // there is text after the end of last element - var endText = new SubString(source, endIdx, source.Length - endIdx); - if (!endText.IsEmptyOrWhitespace()) - { - var abox = CssBox.CreateBox(root); - abox.Text = endText; - } } return root; @@ -96,168 +90,97 @@ public static CssBox ParseDocument(string source) /// Adding box also for text that contains only whitespaces because we don't know yet if /// the box is preformatted. At later stage they will be removed if not relevant. /// - /// the html source to parse - /// the start of the html part - /// the index of the next html tag + /// the html token to parse /// the current box in html tree parsing - private static void AddTextBox(string source, int startIdx, int tagIdx, ref CssBox curBox) + private static void AddTextBox(HtmlDataToken token, ref CssBox curBox) { - var text = tagIdx > startIdx ? new SubString(source, startIdx, tagIdx - startIdx) : null; - if (text != null) - { - var abox = CssBox.CreateBox(curBox); - abox.Text = text; - } + var text = token.Data; + + if (text == null) return; + + var box = CssBox.CreateBox(curBox); + box.Text = text; } + /// /// Parse the html part, the part from prev parsing index to the beginning of the next html tag.
/// - /// the html source to parse - /// the index of the next html tag + /// the html tag token /// the current box in html tree parsing - /// the end of the parsed part, the new start index - private static int ParseHtmlTag(string source, int tagIdx, ref CssBox curBox) + private static void ParseHtmlTag(HtmlTagToken token, ref CssBox curBox) { - var endIdx = source.IndexOf('>', tagIdx + 1); - if (endIdx > 0) + string tagName; + Dictionary tagAttributes; + + if (ParseHtmlTag(token, out tagName, out tagAttributes)) { - string tagName; - Dictionary tagAttributes; - var length = endIdx - tagIdx + 1 - (source[endIdx - 1] == '/' ? 1 : 0); - if (ParseHtmlTag(source, tagIdx, length, out tagName, out tagAttributes)) + if (!HtmlUtils.IsSingleTag(tagName.ToLowerInvariant()) && curBox.ParentBox != null) { - if (!HtmlUtils.IsSingleTag(tagName) && curBox.ParentBox != null) - { - // need to find the parent tag to go one level up - curBox = DomUtils.FindParent(curBox.ParentBox, tagName, curBox); - } + // need to find the parent tag to go one level up + curBox = CloseElement(curBox, tagName); } - else if (!string.IsNullOrEmpty(tagName)) + } + else if (!string.IsNullOrEmpty(tagName)) + { + while (true) { - //new SubString(source, lastEnd + 1, tagmatch.Index - lastEnd - 1) - var isSingle = HtmlUtils.IsSingleTag(tagName) || source[endIdx - 1] == '/'; - var tag = new HtmlTag(tagName, isSingle, tagAttributes); - - if (isSingle) + if (curBox.HtmlTag != null && HtmlUtils.CanEndTagBeOmitted(curBox.HtmlTag.Name.ToLowerInvariant(), tagName.ToLowerInvariant())) { - // the current box is not changed - CssBox.CreateBox(tag, curBox); + curBox = CloseElement(curBox, curBox.HtmlTag.Name); } else { - // go one level down, make the new box the current box - curBox = CssBox.CreateBox(tag, curBox); + break; } } + + var isSingle = HtmlUtils.IsSingleTag(tagName.ToLowerInvariant()) || token.IsEmptyElement; + var tag = new HtmlTag(tagName, isSingle, tagAttributes); + + if (isSingle) + { + // the current box is not changed + CssBox.CreateBox(tag, curBox); + } else { - endIdx = tagIdx + 1; + // go one level down, make the new box the current box + curBox = CssBox.CreateBox(tag, curBox); } } - return endIdx; } - /// - /// Parse raw html tag source to object.
- /// Extract attributes found on the tag. - /// - /// the html source to parse - /// the start index of the tag in the source - /// the length of the tag from the start index in the source - /// return the name of the html tag - /// return the dictionary of tag attributes - /// true - the tag is closing tag, false - otherwise - private static bool ParseHtmlTag(string source, int idx, int length, out string name, out Dictionary attributes) + private static CssBox CloseElement(CssBox cssBox, string tagName) { - idx++; - length = length - (source[idx + length - 3] == '/' ? 3 : 2); - - // Check if is end tag - var isClosing = false; - if (source[idx] == '/') - { - idx++; - length--; - isClosing = true; - } - - int spaceIdx = idx; - while (spaceIdx < idx + length && !char.IsWhiteSpace(source, spaceIdx)) - spaceIdx++; - - // Get the name of the tag - name = source.Substring(idx, spaceIdx - idx).ToLower(); - - attributes = null; - if (!isClosing && idx + length > spaceIdx) - { - ExtractAttributes(source, spaceIdx, length - (spaceIdx - idx), out attributes); - } - - return isClosing; + return DomUtils.FindParent(cssBox.ParentBox, tagName, cssBox); } /// - /// Extract html tag attributes from the given sub-string. + /// /// - /// the html source to parse - /// the start index of the tag attributes in the source - /// the length of the tag attributes from the start index in the source - /// return the dictionary of tag attributes - private static void ExtractAttributes(string source, int idx, int length, out Dictionary attributes) + /// + /// + /// + /// + private static bool ParseHtmlTag(HtmlTagToken token, out string name, out Dictionary attributes) { - attributes = null; - - int startIdx = idx; - while (startIdx < idx + length) - { - while (startIdx < idx + length && char.IsWhiteSpace(source, startIdx)) - startIdx++; + var isClosing = token.IsEndTag; - var endIdx = startIdx + 1; - while (endIdx < idx + length && !char.IsWhiteSpace(source, endIdx) && source[endIdx] != '=') - endIdx++; + name = token.Name; - if (startIdx < idx + length) - { - var key = source.Substring(startIdx, endIdx - startIdx); - var value = ""; - - startIdx = endIdx + 1; - while (startIdx < idx + length && (char.IsWhiteSpace(source, startIdx) || source[startIdx] == '=')) - startIdx++; - - bool hasPChar = false; - if (startIdx < idx + length) - { - char pChar = source[startIdx]; - if (pChar == '"' || pChar == '\'') - { - hasPChar = true; - startIdx++; - } - - endIdx = startIdx + (hasPChar ? 0 : 1); - while (endIdx < idx + length && (hasPChar ? source[endIdx] != pChar : !char.IsWhiteSpace(source, endIdx))) - endIdx++; - - value = source.Substring(startIdx, endIdx - startIdx); - value = HtmlUtils.DecodeHtml(value); - } - - if (key.Length != 0) - { - if (attributes == null) - attributes = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - attributes[key.ToLower()] = value; - } + attributes = null; - startIdx = endIdx + (hasPChar ? 2 : 1); - } + if (!isClosing) + { + attributes = token.Attributes + .GroupBy(x => x.Name) + .ToDictionary(x => x.Key, x => x.First().Value); } + + return isClosing; } #endregion } -} \ No newline at end of file +} diff --git a/Source/HtmlRenderer/Core/Utils/CommonUtils.cs b/Source/HtmlRenderer/Core/Utils/CommonUtils.cs index 12c03ad32..8da8bad95 100644 --- a/Source/HtmlRenderer/Core/Utils/CommonUtils.cs +++ b/Source/HtmlRenderer/Core/Utils/CommonUtils.cs @@ -96,6 +96,17 @@ public static bool IsAsianCharecter(char ch) return ch >= 0x4e00 && ch <= 0xFA2D; } + /// + /// Check if the given string is non-empty and contains only whitespace characters.
+ /// Unlike , this returns false for an empty string. + /// + /// the string to check + /// true - non-empty and all whitespace, false - otherwise + public static bool IsNonEmptyWhitespace(string s) + { + return s.Length > 0 && string.IsNullOrWhiteSpace(s); + } + /// /// Check if the given char is a digit character (0-9) and (0-9, a-f for HEX) /// diff --git a/Source/HtmlRenderer/Core/Utils/DomUtils.cs b/Source/HtmlRenderer/Core/Utils/DomUtils.cs index 6d00a4abc..ba0fc8af3 100644 --- a/Source/HtmlRenderer/Core/Utils/DomUtils.cs +++ b/Source/HtmlRenderer/Core/Utils/DomUtils.cs @@ -112,6 +112,50 @@ public static CssBox GetPreviousSibling(CssBox b) return null; } + /// + /// Get all following siblings of the given box that match the given predicate.
+ /// If is true, stops at the first non-matching sibling. + /// + /// the box to get the following siblings of + /// predicate to filter siblings by + /// true - stop enumeration at the first non-matching sibling, false - skip non-matching siblings + /// matching following siblings + public static IEnumerable GetFollowingSiblings(CssBox box, Predicate matcher, bool isConsecutive) + { + if (box.ParentBox == null) yield break; + + var index = box.ParentBox.Boxes.IndexOf(box); + const int diff = 1; + + while (box.ParentBox.Boxes.Count > index + diff) + { + var sib = box.ParentBox.Boxes[index + diff]; + + if (matcher(sib)) + { + yield return sib; + } + else if (isConsecutive) + { + yield break; + } + + index += diff; + } + } + + /// + /// Check if the given box is a "proper table child" per https://www.w3.org/TR/CSS2/tables.html#anonymous-boxes + /// + /// the box to check + /// true - proper table child, false - otherwise + public static bool IsProperTableChild(CssBox box) + { + return box.IsTableRowGroupBox || box.Display == CssConstants.TableRow || + box.Display == CssConstants.TableColumn || box.Display == CssConstants.TableColumnGroup || + box.Display == CssConstants.TableCaption; + } + /// /// Gets the previous sibling of this box. /// @@ -502,7 +546,7 @@ private static int GetSelectedPlainText(StringBuilder sb, CssBox box) } // empty span box - if (box.Boxes.Count < 1 && box.Text != null && box.Text.IsWhitespace()) + if (box.Boxes.Count < 1 && box.Text != null && CommonUtils.IsNonEmptyWhitespace(box.Text)) { sb.Append(' '); } diff --git a/Source/HtmlRenderer/Core/Utils/HtmlConstants.cs b/Source/HtmlRenderer/Core/Utils/HtmlConstants.cs index 834cbe540..c69623a5c 100644 --- a/Source/HtmlRenderer/Core/Utils/HtmlConstants.cs +++ b/Source/HtmlRenderer/Core/Utils/HtmlConstants.cs @@ -64,7 +64,7 @@ internal static class HtmlConstants // public const string MENU = "MENU"; // public const string META = "META"; // public const string NOFRAMES = "NOFRAMES"; - // public const string NOSCRIPT = "NOSCRIPT"; + public const string NoScript = "noscript"; // public const string OBJECT = "OBJECT"; // public const string OL = "OL"; // public const string OPTGROUP = "OPTGROUP"; diff --git a/Source/HtmlRenderer/Core/Utils/HtmlUtils.cs b/Source/HtmlRenderer/Core/Utils/HtmlUtils.cs index d0198d20c..5b24fa99b 100644 --- a/Source/HtmlRenderer/Core/Utils/HtmlUtils.cs +++ b/Source/HtmlRenderer/Core/Utils/HtmlUtils.cs @@ -310,6 +310,84 @@ public static bool IsSingleTag(string tagName) return _list.Contains(tagName); } + // https://html.spec.whatwg.org/dev/dom.html#concept-element-tag-omission + public static bool CanEndTagBeOmitted(string currentTagName, string tagName) + { + switch (currentTagName) + { + case "p": + return ShouldTagCloseParagraph(tagName); + case "td": + return ShouldTagCloseTableCell(tagName); + case "tr": + return ShouldTagCloseTableRow(tagName); + default: + return false; + } + } + + // Close

tags per https://html.spec.whatwg.org/dev/grouping-content.html#the-p-element + public static bool ShouldTagCloseParagraph(string tagName) + { + switch (tagName) + { + case "address": + case "article": + case "aside": + case "blockquote": + case "details": + case "dialog": + case "div": + case "dl": + case "fieldset": + case "figcaption": + case "figure": + case "footer": + case "form": + case "h1": + case "h2": + case "h3": + case "h4": + case "h5": + case "h6": + case "header": + case "hgroup": + case "hr": + case "main": + case "menu": + case "nav": + case "ol": + case "p": + case "pre": + case "search": + case "section": + case "table": + case "ul": + return true; + default: + return false; + } + } + + // Close tags per https://html.spec.whatwg.org/dev/tables.html#the-td-element + public static bool ShouldTagCloseTableCell(string tagName) + { + switch (tagName) + { + case "td": + case "th": + case "tr": + return true; + default: + return false; + } + } + + public static bool ShouldTagCloseTableRow(string tagName) + { + return tagName == "tr"; + } + ///

/// Decode html encoded string to regular string.
/// Handles <, >, "&. diff --git a/Source/HtmlRenderer/Core/Utils/SubString.cs b/Source/HtmlRenderer/Core/Utils/SubString.cs deleted file mode 100644 index f06f740f2..000000000 --- a/Source/HtmlRenderer/Core/Utils/SubString.cs +++ /dev/null @@ -1,187 +0,0 @@ -// "Therefore those skilled at the unorthodox -// are infinite as heaven and earth, -// inexhaustible as the great rivers. -// When they come to an end, -// they begin again, -// like the days and months; -// they die and are reborn, -// like the four seasons." -// -// - Sun Tsu, -// "The Art of War" - -using System; - -namespace TheArtOfDev.HtmlRenderer.Core.Utils -{ - /// - /// Represents sub-string of a full string starting at specific location with a specific length. - /// - internal sealed class SubString - { - #region Fields and Consts - - /// - /// the full string that this sub-string is part of - /// - private readonly string _fullString; - - /// - /// the start index of the sub-string - /// - private readonly int _startIdx; - - /// - /// the length of the sub-string starting at - /// - private readonly int _length; - - #endregion - - - /// - /// Init sub-string that is the full string. - /// - /// the full string that this sub-string is part of - public SubString(string fullString) - { - ArgChecker.AssertArgNotNull(fullString, "fullString"); - - _fullString = fullString; - _startIdx = 0; - _length = fullString.Length; - } - - /// - /// Init. - /// - /// the full string that this sub-string is part of - /// the start index of the sub-string - /// the length of the sub-string starting at - /// is null - public SubString(string fullString, int startIdx, int length) - { - ArgChecker.AssertArgNotNull(fullString, "fullString"); - if (startIdx < 0 || startIdx >= fullString.Length) - throw new ArgumentOutOfRangeException("startIdx", "Must within fullString boundries"); - if (length < 0 || startIdx + length > fullString.Length) - throw new ArgumentOutOfRangeException("length", "Must within fullString boundries"); - - _fullString = fullString; - _startIdx = startIdx; - _length = length; - } - - /// - /// the full string that this sub-string is part of - /// - public string FullString - { - get { return _fullString; } - } - - /// - /// the start index of the sub-string - /// - public int StartIdx - { - get { return _startIdx; } - } - - /// - /// the length of the sub-string starting at - /// - public int Length - { - get { return _length; } - } - - /// - /// Get string char at specific index. - /// - /// the idx to get the char at - /// char at index - public char this[int idx] - { - get - { - if (idx < 0 || idx > _length) - throw new ArgumentOutOfRangeException("idx", "must be within the string range"); - return _fullString[_startIdx + idx]; - } - } - - /// - /// Is the sub-string is empty string. - /// - /// true - empty string, false - otherwise - public bool IsEmpty() - { - return _length < 1; - } - - /// - /// Is the sub-string is empty string or contains only whitespaces. - /// - /// true - empty or whitespace string, false - otherwise - public bool IsEmptyOrWhitespace() - { - for (int i = 0; i < _length; i++) - { - if (!char.IsWhiteSpace(_fullString, _startIdx + i)) - return false; - } - return true; - } - - /// - /// Is the sub-string contains only whitespaces (at least one). - /// - /// true - empty or whitespace string, false - otherwise - public bool IsWhitespace() - { - if (_length < 1) - return false; - for (int i = 0; i < _length; i++) - { - if (!char.IsWhiteSpace(_fullString, _startIdx + i)) - return false; - } - return true; - } - - /// - /// Get a string of the sub-string.
- /// This will create a new string object! - /// - /// new string that is the sub-string represented by this instance - public string CutSubstring() - { - return _length > 0 ? _fullString.Substring(_startIdx, _length) : string.Empty; - } - - /// - /// Retrieves a substring from this instance. The substring starts at a specified character position and has a specified length. - /// - /// The zero-based starting character position of a substring in this instance. - /// The number of characters in the substring. - /// A String equivalent to the substring of length length that begins at startIndex in this instance, or - /// Empty if startIndex is equal to the length of this instance and length is zero. - public string Substring(int startIdx, int length) - { - if (startIdx < 0 || startIdx > _length) - throw new ArgumentOutOfRangeException("startIdx"); - if (length > _length) - throw new ArgumentOutOfRangeException("length"); - if (startIdx + length > _length) - throw new ArgumentOutOfRangeException("length"); - - return _fullString.Substring(_startIdx + startIdx, length); - } - - public override string ToString() - { - return string.Format("Sub-string: {0}", _length > 0 ? _fullString.Substring(_startIdx, _length) : string.Empty); - } - } -} \ No newline at end of file diff --git a/Source/HtmlRenderer/HtmlRenderer.csproj b/Source/HtmlRenderer/HtmlRenderer.csproj index 273dcccf8..0850eeb90 100644 --- a/Source/HtmlRenderer/HtmlRenderer.csproj +++ b/Source/HtmlRenderer/HtmlRenderer.csproj @@ -27,5 +27,9 @@ For existing implementations see: HtmlRenderer.WinForms, HtmlRenderer.WPF and Ht + + + + \ No newline at end of file From 8fd853cda86d936155823014d3bf81a90176fb82 Mon Sep 17 00:00:00 2001 From: Justin Haygood Date: Mon, 13 Jul 2026 22:22:08 -0400 Subject: [PATCH 2/3] Polish HTML5 parsing script sample Improve the HTML5 parsing demo snippet by expanding the inline `if` into a multi-line block and refining the demo string text. The escaped source example and live script output were updated together to stay consistent and easier to read. --- Source/Demo/Common/Samples/15.HTML5 Parsing.htm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Source/Demo/Common/Samples/15.HTML5 Parsing.htm b/Source/Demo/Common/Samples/15.HTML5 Parsing.htm index bfa8df0a1..327872352 100644 --- a/Source/Demo/Common/Samples/15.HTML5 Parsing.htm +++ b/Source/Demo/Common/Samples/15.HTML5 Parsing.htm @@ -94,13 +94,17 @@

<script> content is not mistaken for markup

Source:
<script>
-    if (1 < 2 && 3 > 2) { runDemo('<b>this looks like a tag but is just a string</b>'); }
+    if (1 < 2 && 3 > 2) {
+        runDemo('<b>looks like a tag</b>, but is just a string');
+    }
 </script>
 <p>This paragraph still renders correctly right after the script block.</p>
Live result:

This paragraph still renders correctly right after the script block.

From 2445dc603fe1ebd0637b790ee084ff76b5d1580a Mon Sep 17 00:00:00 2001 From: Justin Haygood Date: Mon, 13 Jul 2026 22:23:11 -0400 Subject: [PATCH 3/3] Remove HtmlParserVerify internals exposure Delete the `InternalsVisibleTo` entry for `HtmlParserVerify` from `HtmlRenderer.csproj`, narrowing assembly internals access and keeping project metadata aligned with current test/verification setup. --- Source/HtmlRenderer/HtmlRenderer.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/HtmlRenderer/HtmlRenderer.csproj b/Source/HtmlRenderer/HtmlRenderer.csproj index 0850eeb90..e55706cf6 100644 --- a/Source/HtmlRenderer/HtmlRenderer.csproj +++ b/Source/HtmlRenderer/HtmlRenderer.csproj @@ -29,7 +29,4 @@ For existing implementations see: HtmlRenderer.WinForms, HtmlRenderer.WPF and Ht - - - \ No newline at end of file