From f01eeaffd4e14981bb9e58ef3fe000bba790ecb4 Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 12:13:45 +0300 Subject: [PATCH 1/9] feat: implement DOCX numbered headings extraction and application to sections and markdown --- internal/parser/parser/docx_numbering.go | 635 ++++++++++++++++++ internal/parser/parser/docx_numbering_test.go | 110 +++ internal/parser/parser/docx_parser.go | 5 + 3 files changed, 750 insertions(+) create mode 100644 internal/parser/parser/docx_numbering.go create mode 100644 internal/parser/parser/docx_numbering_test.go diff --git a/internal/parser/parser/docx_numbering.go b/internal/parser/parser/docx_numbering.go new file mode 100644 index 00000000000..1eea6c8ed8b --- /dev/null +++ b/internal/parser/parser/docx_numbering.go @@ -0,0 +1,635 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package parser + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "io" + "strconv" + "strings" +) + +type docxNumberedHeading struct { + Text string + NumberedText string + Level int +} + +type docxXMLValue struct { + Value string `xml:"val,attr"` +} + +type docxXMLNumberingRef struct { + Level *docxXMLValue `xml:"ilvl"` + ID *docxXMLValue `xml:"numId"` +} + +type docxXMLParagraphProperties struct { + Style *docxXMLValue `xml:"pStyle"` + NumberingRef *docxXMLNumberingRef `xml:"numPr"` + OutlineLevel *docxXMLValue `xml:"outlineLvl"` +} + +type docxXMLParagraph struct { + Properties *docxXMLParagraphProperties `xml:"pPr"` + InnerXML []byte `xml:",innerxml"` +} + +type docxXMLStyle struct { + ID string `xml:"styleId,attr"` + Name *docxXMLValue `xml:"name"` + BasedOn *docxXMLValue `xml:"basedOn"` + Properties *docxXMLParagraphProperties `xml:"pPr"` +} + +type docxXMLStyles struct { + Styles []docxXMLStyle `xml:"style"` +} + +type docxXMLNumberingLevel struct { + Level int `xml:"ilvl,attr"` + Start *docxXMLValue `xml:"start"` + Format *docxXMLValue `xml:"numFmt"` + Text *docxXMLValue `xml:"lvlText"` + ParagraphStyle *docxXMLValue `xml:"pStyle"` +} + +type docxXMLAbstractNumbering struct { + ID int `xml:"abstractNumId,attr"` + Levels []docxXMLNumberingLevel `xml:"lvl"` +} + +type docxXMLLevelOverride struct { + Level int `xml:"ilvl,attr"` + StartOverride *docxXMLValue `xml:"startOverride"` + LevelDef *docxXMLNumberingLevel `xml:"lvl"` +} + +type docxXMLNumberingInstance struct { + ID int `xml:"numId,attr"` + AbstractID *docxXMLValue `xml:"abstractNumId"` + Overrides []docxXMLLevelOverride `xml:"lvlOverride"` +} + +type docxXMLNumbering struct { + Abstracts []docxXMLAbstractNumbering `xml:"abstractNum"` + Instances []docxXMLNumberingInstance `xml:"num"` +} + +type docxNumberingLevel struct { + Start int + Format string + Text string + ParagraphStyle string +} + +type docxNumberingInstance struct { + AbstractID int + Overrides map[int]docxNumberingLevel +} + +type docxNumberingDefinitions struct { + Abstracts map[int]map[int]docxNumberingLevel + Instances map[int]docxNumberingInstance + Order []int +} + +type docxParagraphNumbering struct { + ID int + Level int +} + +type docxResolvedStyle struct { + Name string + NumberingRef *docxParagraphNumbering + OutlineLevel *int +} + +// extractDOCXNumberedHeadings materializes Word's display-only heading +// numbers. Word stores paragraph text in document.xml and the visible marker +// separately in numbering.xml, so office_oxide's text runs cannot contain it. +func extractDOCXNumberedHeadings(data []byte) []docxNumberedHeading { + parts := readDOCXXMLParts(data, "word/document.xml", "word/styles.xml", "word/numbering.xml") + if len(parts["word/document.xml"]) == 0 || len(parts["word/numbering.xml"]) == 0 { + return nil + } + + paragraphs := parseDOCXParagraphs(parts["word/document.xml"]) + styles := parseDOCXStyles(parts["word/styles.xml"]) + numbering := parseDOCXNumbering(parts["word/numbering.xml"]) + if len(paragraphs) == 0 || len(numbering.Instances) == 0 { + return nil + } + + counters := make(map[int][]int) + var headings []docxNumberedHeading + for _, paragraph := range paragraphs { + styleID := "" + if paragraph.Properties != nil && paragraph.Properties.Style != nil { + styleID = paragraph.Properties.Style.Value + } + style := resolveDOCXStyle(styleID, styles, nil) + ref := paragraphNumberingRef(paragraph.Properties) + if ref == nil { + ref = style.NumberingRef + } + if ref == nil { + ref = numberingRefForParagraphStyle(styleID, numbering) + } + if ref == nil { + continue + } + + level, ok := numbering.resolveLevel(ref.ID, ref.Level) + if !ok || level.Format == "bullet" || level.Format == "none" || level.Text == "" { + continue + } + values := counters[ref.ID] + if len(values) < 9 { + values = make([]int, 9) + } + if values[ref.Level] == 0 { + values[ref.Level] = level.Start + } else { + values[ref.Level]++ + } + for i := ref.Level + 1; i < len(values); i++ { + values[i] = 0 + } + counters[ref.ID] = values + + text := paragraphText(paragraph) + headingLevel, isHeading := docxHeadingLevel(styleID, style, paragraph.Properties) + if !isHeading || strings.TrimSpace(text) == "" { + continue + } + marker := renderDOCXNumberingMarker(level.Text, ref.ID, values, numbering) + marker = strings.TrimSpace(marker) + if marker == "" { + continue + } + numberedText := strings.TrimSpace(text) + if !strings.HasPrefix(numberedText, marker) { + numberedText = marker + " " + numberedText + } + headings = append(headings, docxNumberedHeading{ + Text: strings.TrimSpace(text), + NumberedText: numberedText, + Level: headingLevel, + }) + } + return headings +} + +func readDOCXXMLParts(data []byte, names ...string) map[string][]byte { + wanted := make(map[string]bool, len(names)) + for _, name := range names { + wanted[name] = true + } + parts := make(map[string][]byte, len(names)) + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return parts + } + for _, file := range zr.File { + if !wanted[file.Name] { + continue + } + r, err := file.Open() + if err != nil { + continue + } + content, err := io.ReadAll(r) + r.Close() + if err == nil { + parts[file.Name] = content + } + } + return parts +} + +func parseDOCXParagraphs(data []byte) []docxXMLParagraph { + decoder := xml.NewDecoder(bytes.NewReader(data)) + var paragraphs []docxXMLParagraph + for { + token, err := decoder.Token() + if err != nil { + break + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "p" { + continue + } + var paragraph docxXMLParagraph + if err := decoder.DecodeElement(¶graph, &start); err != nil { + break + } + paragraphs = append(paragraphs, paragraph) + } + return paragraphs +} + +func parseDOCXStyles(data []byte) map[string]docxXMLStyle { + var document docxXMLStyles + _ = xml.Unmarshal(data, &document) + styles := make(map[string]docxXMLStyle, len(document.Styles)) + for _, style := range document.Styles { + styles[style.ID] = style + } + return styles +} + +func parseDOCXNumbering(data []byte) docxNumberingDefinitions { + var document docxXMLNumbering + _ = xml.Unmarshal(data, &document) + definitions := docxNumberingDefinitions{ + Abstracts: make(map[int]map[int]docxNumberingLevel), + Instances: make(map[int]docxNumberingInstance), + } + for _, abstract := range document.Abstracts { + levels := make(map[int]docxNumberingLevel, len(abstract.Levels)) + for _, level := range abstract.Levels { + levels[level.Level] = convertDOCXNumberingLevel(level) + } + definitions.Abstracts[abstract.ID] = levels + } + for _, instance := range document.Instances { + abstractID, ok := parseDOCXXMLInt(instance.AbstractID) + if !ok { + continue + } + resolved := docxNumberingInstance{AbstractID: abstractID, Overrides: make(map[int]docxNumberingLevel)} + for _, override := range instance.Overrides { + level, exists := definitions.Abstracts[abstractID][override.Level] + if override.LevelDef != nil { + level = convertDOCXNumberingLevel(*override.LevelDef) + exists = true + } + if !exists { + continue + } + if start, ok := parseDOCXXMLInt(override.StartOverride); ok { + level.Start = start + } + resolved.Overrides[override.Level] = level + } + definitions.Instances[instance.ID] = resolved + definitions.Order = append(definitions.Order, instance.ID) + } + return definitions +} + +func convertDOCXNumberingLevel(level docxXMLNumberingLevel) docxNumberingLevel { + start, ok := parseDOCXXMLInt(level.Start) + if !ok { + start = 1 + } + result := docxNumberingLevel{Start: start} + if level.Format != nil { + result.Format = level.Format.Value + } + if level.Text != nil { + result.Text = level.Text.Value + } + if level.ParagraphStyle != nil { + result.ParagraphStyle = level.ParagraphStyle.Value + } + return result +} + +func (definitions docxNumberingDefinitions) resolveLevel(id, level int) (docxNumberingLevel, bool) { + instance, ok := definitions.Instances[id] + if !ok { + return docxNumberingLevel{}, false + } + if override, ok := instance.Overrides[level]; ok { + return override, true + } + resolved, ok := definitions.Abstracts[instance.AbstractID][level] + return resolved, ok +} + +func paragraphNumberingRef(properties *docxXMLParagraphProperties) *docxParagraphNumbering { + if properties == nil || properties.NumberingRef == nil { + return nil + } + id, ok := parseDOCXXMLInt(properties.NumberingRef.ID) + if !ok || id == 0 { + return nil + } + level, _ := parseDOCXXMLInt(properties.NumberingRef.Level) + return &docxParagraphNumbering{ID: id, Level: level} +} + +func resolveDOCXStyle(id string, styles map[string]docxXMLStyle, seen map[string]bool) docxResolvedStyle { + if id == "" { + return docxResolvedStyle{} + } + if seen == nil { + seen = make(map[string]bool) + } + if seen[id] { + return docxResolvedStyle{} + } + seen[id] = true + style, ok := styles[id] + if !ok { + return docxResolvedStyle{} + } + var resolved docxResolvedStyle + if style.BasedOn != nil { + resolved = resolveDOCXStyle(style.BasedOn.Value, styles, seen) + } + if style.Name != nil { + resolved.Name = style.Name.Value + } + if ref := paragraphNumberingRef(style.Properties); ref != nil { + resolved.NumberingRef = ref + } + if style.Properties != nil { + if level, ok := parseDOCXXMLInt(style.Properties.OutlineLevel); ok { + resolved.OutlineLevel = &level + } + } + return resolved +} + +func numberingRefForParagraphStyle(styleID string, definitions docxNumberingDefinitions) *docxParagraphNumbering { + if styleID == "" { + return nil + } + for _, id := range definitions.Order { + instance := definitions.Instances[id] + for level, definition := range definitions.Abstracts[instance.AbstractID] { + if definition.ParagraphStyle == styleID { + return &docxParagraphNumbering{ID: id, Level: level} + } + } + } + return nil +} + +func docxHeadingLevel(styleID string, style docxResolvedStyle, properties *docxXMLParagraphProperties) (int, bool) { + if properties != nil { + if level, ok := parseDOCXXMLInt(properties.OutlineLevel); ok && level >= 0 && level < 9 { + return level + 1, true + } + } + if style.OutlineLevel != nil && *style.OutlineLevel >= 0 && *style.OutlineLevel < 9 { + return *style.OutlineLevel + 1, true + } + for _, name := range []string{styleID, style.Name} { + lower := strings.ToLower(strings.ReplaceAll(name, " ", "")) + for _, prefix := range []string{"heading", "заголовок"} { + if !strings.HasPrefix(lower, prefix) { + continue + } + level, err := strconv.Atoi(strings.TrimPrefix(lower, prefix)) + if err == nil && level >= 1 && level <= 9 { + return level, true + } + } + } + return 0, false +} + +func paragraphText(paragraph docxXMLParagraph) string { + decoder := xml.NewDecoder(bytes.NewReader(paragraph.InnerXML)) + var builder strings.Builder + for { + token, err := decoder.Token() + if err != nil { + break + } + start, ok := token.(xml.StartElement) + if !ok { + continue + } + switch start.Name.Local { + case "t": + var text string + if err := decoder.DecodeElement(&text, &start); err == nil { + builder.WriteString(text) + } + case "tab": + builder.WriteByte('\t') + } + } + return builder.String() +} + +func renderDOCXNumberingMarker(template string, id int, counters []int, definitions docxNumberingDefinitions) string { + marker := template + for index := 0; index < 9; index++ { + placeholder := "%" + strconv.Itoa(index+1) + if !strings.Contains(marker, placeholder) { + continue + } + value := counters[index] + level, ok := definitions.resolveLevel(id, index) + if value == 0 && ok { + value = level.Start + } + format := "decimal" + if ok && level.Format != "" { + format = level.Format + } + marker = strings.ReplaceAll(marker, placeholder, formatDOCXNumber(value, format)) + } + return marker +} + +func formatDOCXNumber(value int, format string) string { + switch format { + case "lowerLetter": + return formatDOCXLetters(value, false) + case "upperLetter": + return formatDOCXLetters(value, true) + case "lowerRoman": + return strings.ToLower(formatDOCXRoman(value)) + case "upperRoman": + return formatDOCXRoman(value) + case "decimalZero": + if value >= 0 && value < 10 { + return "0" + strconv.Itoa(value) + } + } + return strconv.Itoa(value) +} + +func formatDOCXLetters(value int, upper bool) string { + if value <= 0 { + return strconv.Itoa(value) + } + var result []rune + base := 'a' + if upper { + base = 'A' + } + for value > 0 { + value-- + result = append([]rune{base + rune(value%26)}, result...) + value /= 26 + } + return string(result) +} + +func formatDOCXRoman(value int) string { + if value <= 0 || value > 3999 { + return strconv.Itoa(value) + } + values := []struct { + value int + symbol string + }{{1000, "M"}, {900, "CM"}, {500, "D"}, {400, "CD"}, {100, "C"}, {90, "XC"}, {50, "L"}, {40, "XL"}, {10, "X"}, {9, "IX"}, {5, "V"}, {4, "IV"}, {1, "I"}} + var builder strings.Builder + for _, item := range values { + for value >= item.value { + builder.WriteString(item.symbol) + value -= item.value + } + } + return builder.String() +} + +func parseDOCXXMLInt(value *docxXMLValue) (int, bool) { + if value == nil { + return 0, false + } + parsed, err := strconv.Atoi(value.Value) + return parsed, err == nil +} + +func applyDOCXNumberedHeadingsToSections(items []map[string]any, headings []docxNumberedHeading) []map[string]any { + if len(headings) == 0 { + return items + } + result := make([]map[string]any, 0, len(items)) + headingIndex := 0 + for _, item := range items { + text, ok := item["text"].(string) + if !ok || text == "" { + result = append(result, item) + continue + } + lines := strings.Split(text, "\n") + matches := make([]int, len(lines)) + matched := false + searchFrom := headingIndex + for i, line := range lines { + matches[i] = -1 + for j := searchFrom; j < len(headings); j++ { + if strings.TrimSpace(line) == headings[j].Text { + matches[i] = j + searchFrom = j + 1 + matched = true + break + } + } + } + if !matched { + result = append(result, item) + continue + } + for i, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + copyItem := make(map[string]any, len(item)+1) + for key, value := range item { + copyItem[key] = value + } + if matches[i] >= 0 { + heading := headings[matches[i]] + copyItem["text"] = heading.NumberedText + copyItem["ck_type"] = "heading" + headingIndex = matches[i] + 1 + } else { + copyItem["text"] = line + delete(copyItem, "ck_type") + } + result = append(result, copyItem) + } + } + return result +} + +func applyDOCXNumberedHeadingsToMarkdown(markdown string, headings []docxNumberedHeading) string { + if markdown == "" || len(headings) == 0 { + return markdown + } + lines := strings.Split(markdown, "\n") + headingIndex := 0 + for i, line := range lines { + content, ok := docxMarkdownListOrHeadingText(line) + if !ok { + continue + } + for j := headingIndex; j < len(headings); j++ { + heading := headings[j] + if content != heading.Text { + continue + } + level := heading.Level + if level < 1 { + level = 1 + } + if level > 6 { + level = 6 + } + lines[i] = strings.Repeat("#", level) + " " + heading.NumberedText + headingIndex = j + 1 + break + } + } + return strings.Join(lines, "\n") +} + +func docxMarkdownListOrHeadingText(line string) (string, bool) { + line = strings.TrimSpace(line) + markerEnd := 0 + if strings.HasPrefix(line, "#") { + for markerEnd < len(line) && line[markerEnd] == '#' { + markerEnd++ + } + } else if len(line) >= 2 && (line[0] == '-' || line[0] == '*' || line[0] == '+') && line[1] == ' ' { + markerEnd = 1 + } else { + for markerEnd < len(line) && line[markerEnd] >= '0' && line[markerEnd] <= '9' { + markerEnd++ + } + if markerEnd == 0 || markerEnd >= len(line) || (line[markerEnd] != '.' && line[markerEnd] != ')') { + return "", false + } + markerEnd++ + } + if markerEnd >= len(line) || line[markerEnd] != ' ' { + return "", false + } + content := strings.TrimSpace(line[markerEnd:]) + content = strings.Trim(content, "*_~`") + return content, true +} + +func appendDOCXNumberedHeadingOutlines(outlines []docxOutline, headings []docxNumberedHeading) []docxOutline { + for _, heading := range headings { + outlines = append(outlines, docxOutline{Title: heading.NumberedText, Level: heading.Level - 1}) + } + return outlines +} diff --git a/internal/parser/parser/docx_numbering_test.go b/internal/parser/parser/docx_numbering_test.go new file mode 100644 index 00000000000..d7b1fee4a21 --- /dev/null +++ b/internal/parser/parser/docx_numbering_test.go @@ -0,0 +1,110 @@ +package parser + +import ( + "archive/zip" + "bytes" + "reflect" + "testing" +) + +func numberedHeadingsDOCX(t *testing.T) []byte { + t.Helper() + parts := map[string]string{ + "word/document.xml": ` + + Общие сведения + Установка + Начало работы + Обычный пункт +`, + "word/styles.xml": ` + + + +`, + "word/numbering.xml": ` + + + + + + + +`, + } + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + for name, body := range parts { + file, err := writer.Create(name) + if err != nil { + t.Fatalf("create %s: %v", name, err) + } + if _, err := file.Write([]byte(body)); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close docx: %v", err) + } + return buffer.Bytes() +} + +func TestExtractDOCXNumberedHeadings(t *testing.T) { + got := extractDOCXNumberedHeadings(numberedHeadingsDOCX(t)) + want := []docxNumberedHeading{ + {Text: "Общие сведения", NumberedText: "1 Общие сведения", Level: 1}, + {Text: "Установка", NumberedText: "2 Установка", Level: 1}, + {Text: "Начало работы", NumberedText: "2.1 Начало работы", Level: 2}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("extractDOCXNumberedHeadings mismatch:\n got: %#v\nwant: %#v", got, want) + } +} + +func TestApplyDOCXNumberedHeadingsToSections(t *testing.T) { + headings := extractDOCXNumberedHeadings(numberedHeadingsDOCX(t)) + items := []map[string]any{ + {"text": "Общие сведения\nУстановка\nНачало работы", "image": nil, "doc_type_kwd": "text"}, + {"text": "Обычный текст", "image": nil, "doc_type_kwd": "text"}, + } + got := applyDOCXNumberedHeadingsToSections(items, headings) + want := []map[string]any{ + {"text": "1 Общие сведения", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"}, + {"text": "2 Установка", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"}, + {"text": "2.1 Начало работы", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"}, + {"text": "Обычный текст", "image": nil, "doc_type_kwd": "text"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("applyDOCXNumberedHeadingsToSections mismatch:\n got: %#v\nwant: %#v", got, want) + } +} + +func TestApplyDOCXNumberedHeadingsToMarkdown(t *testing.T) { + headings := extractDOCXNumberedHeadings(numberedHeadingsDOCX(t)) + markdown := "1. Общие сведения\n2. Установка\n 1. Начало работы\n\nОбычный текст" + want := "# 1 Общие сведения\n# 2 Установка\n## 2.1 Начало работы\n\nОбычный текст" + if got := applyDOCXNumberedHeadingsToMarkdown(markdown, headings); got != want { + t.Fatalf("markdown = %q, want %q", got, want) + } +} + +func TestFormatDOCXNumber(t *testing.T) { + tests := []struct { + name string + value int + format string + want string + }{ + {name: "decimal", value: 12, format: "decimal", want: "12"}, + {name: "lower letter", value: 27, format: "lowerLetter", want: "aa"}, + {name: "upper roman", value: 14, format: "upperRoman", want: "XIV"}, + {name: "decimal zero", value: 3, format: "decimalZero", want: "03"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := formatDOCXNumber(test.value, test.format); got != test.want { + t.Fatalf("formatDOCXNumber(%d, %q) = %q, want %q", test.value, test.format, got, test.want) + } + }) + } +} diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go index e6c0972c533..f0a8f64082b 100644 --- a/internal/parser/parser/docx_parser.go +++ b/internal/parser/parser/docx_parser.go @@ -82,6 +82,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data // Extract IR JSON for section building (JSON path) and // embedded-image extraction (both paths). irJSON, irErr := doc.ToIRJSON() + numberedHeadings := extractDOCXNumberedHeadings(data) var figures []DOCXFigure if irErr == nil { figures = extractDOCXFiguresFromIR(irJSON) @@ -96,6 +97,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data } var sections []map[string]any sections = buildDOCXJSONSections(irJSON) + sections = applyDOCXNumberedHeadingsToSections(sections, numberedHeadings) // remove_header_footer: drop sections whose normalized text // matches a docx header/footer entry (mirrors Python // parser.py:889-891 extract_docx_header_footer_texts + @@ -108,6 +110,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data // (mirrors Python parser.py:892-893 remove_toc_word). if p.RemoveTOC { outlines := extractDOCXOutlines(irJSON) + outlines = appendDOCXNumberedHeadingOutlines(outlines, numberedHeadings) sections = removeTOCWord(sections, outlines, isEnglishItems(sections)) } if len(sections) == 0 { @@ -125,6 +128,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data if err != nil { return ParseResult{Err: fmt.Errorf("docx to-markdown: %w", err)} } + md = applyDOCXNumberedHeadingsToMarkdown(md, numberedHeadings) // remove_header_footer on Markdown: filter lines by exact match // (mirrors Python parser.py:923-926 split lines → filter → rejoin). if p.RemoveHeaderFooter { @@ -145,6 +149,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data // (mirrors Python parser.py:927-928 remove_toc_word on Markdown). if p.RemoveTOC && irErr == nil { outlines := extractDOCXOutlines(irJSON) + outlines = appendDOCXNumberedHeadingOutlines(outlines, numberedHeadings) lines := strings.Split(md, "\n") lineItems := make([]map[string]any, 0, len(lines)) for _, ln := range lines { From 1cd0cf762f0b7258cf35d2bbd6b286d8d66b132c Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 12:54:32 +0300 Subject: [PATCH 2/9] feat: add DOCX numbering resolver and integrate numbered headings extraction for markdown --- deepdoc/parser/docx_numbering.py | 333 ++++++++++++++++++ deepdoc/parser/docx_parser.py | 8 +- rag/app/naive.py | 9 +- rag/flow/parser/utils.py | 12 +- .../deepdoc/parser/test_docx_numbering.py | 96 +++++ 5 files changed, 450 insertions(+), 8 deletions(-) create mode 100644 deepdoc/parser/docx_numbering.py create mode 100644 test/unit_test/deepdoc/parser/test_docx_numbering.py diff --git a/deepdoc/parser/docx_numbering.py b/deepdoc/parser/docx_numbering.py new file mode 100644 index 00000000000..60ea1d7e9da --- /dev/null +++ b/deepdoc/parser/docx_numbering.py @@ -0,0 +1,333 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from dataclasses import dataclass + +from docx.oxml.ns import qn + + +@dataclass(frozen=True) +class NumberedHeading: + text: str + numbered_text: str + level: int + + +@dataclass(frozen=True) +class _NumberingLevel: + start: int = 1 + number_format: str = "decimal" + level_text: str = "" + paragraph_style: str = "" + + +@dataclass(frozen=True) +class _NumberingInstance: + abstract_id: int + overrides: dict[int, _NumberingLevel] + + +class DOCXNumberingResolver: + """Materialize display-only Word numbering while paragraphs are visited.""" + + def __init__(self, document): + self._abstracts: dict[int, dict[int, _NumberingLevel]] = {} + self._instances: dict[int, _NumberingInstance] = {} + self._instance_order: list[int] = [] + self._counters: dict[int, list[int]] = {} + self._parse_numbering(document) + + def numbered_heading(self, paragraph) -> NumberedHeading | None: + """Advance numbering state and return a materialized heading, if any.""" + reference = self._paragraph_numbering_reference(paragraph) + if reference is None: + return None + + number_id, list_level = reference + definition = self._resolve_level(number_id, list_level) + if definition is None or definition.number_format in {"bullet", "none"} or not definition.level_text: + return None + + values = self._counters.setdefault(number_id, [0] * 9) + values[list_level] = definition.start if values[list_level] == 0 else values[list_level] + 1 + for index in range(list_level + 1, len(values)): + values[index] = 0 + + heading_level = self._heading_level(paragraph) + text = paragraph.text.strip() + if heading_level is None or not text: + return None + + marker = self._render_marker(definition.level_text, number_id, values).strip() + if not marker: + return None + numbered_text = text if text == marker or text.startswith(f"{marker} ") else f"{marker} {text}" + return NumberedHeading(text=text, numbered_text=numbered_text, level=heading_level) + + def heading_level(self, paragraph) -> int | None: + """Return the one-based outline level for a heading paragraph.""" + return self._heading_level(paragraph) + + def _parse_numbering(self, document): + try: + root = document.part.numbering_part.element + except (AttributeError, KeyError): + return + + for abstract in root.findall(qn("w:abstractNum")): + abstract_id = _int_attribute(abstract, "w:abstractNumId") + if abstract_id is None: + continue + levels = {} + for level_element in abstract.findall(qn("w:lvl")): + list_level = _int_attribute(level_element, "w:ilvl") + if list_level is not None: + levels[list_level] = _parse_level(level_element) + self._abstracts[abstract_id] = levels + + for number in root.findall(qn("w:num")): + number_id = _int_attribute(number, "w:numId") + abstract_id = _child_int(number, "w:abstractNumId") + if number_id is None or abstract_id is None: + continue + overrides = {} + for override in number.findall(qn("w:lvlOverride")): + list_level = _int_attribute(override, "w:ilvl") + if list_level is None: + continue + definition = self._abstracts.get(abstract_id, {}).get(list_level) + level_element = override.find(qn("w:lvl")) + if level_element is not None: + definition = _parse_level(level_element) + if definition is None: + continue + start_override = _child_int(override, "w:startOverride") + if start_override is not None: + definition = _NumberingLevel( + start=start_override, + number_format=definition.number_format, + level_text=definition.level_text, + paragraph_style=definition.paragraph_style, + ) + overrides[list_level] = definition + self._instances[number_id] = _NumberingInstance(abstract_id=abstract_id, overrides=overrides) + self._instance_order.append(number_id) + + def _paragraph_numbering_reference(self, paragraph): + reference = _properties_numbering_reference(paragraph._p.pPr) + if reference is not None: + return reference + + style = paragraph.style + seen = set() + while style is not None and style.style_id not in seen: + seen.add(style.style_id) + reference = _properties_numbering_reference(style.element.pPr) + if reference is not None: + number_id, list_level = reference + if style.element.pPr.numPr is not None and style.element.pPr.numPr.ilvl is None: + list_level = self._level_for_style(number_id, style.style_id, list_level) + return number_id, list_level + style = style.base_style + + style_id = paragraph.style.style_id if paragraph.style is not None else "" + for number_id in self._instance_order: + list_level = self._level_for_style(number_id, style_id, None) + if list_level is not None: + return number_id, list_level + return None + + def _level_for_style(self, number_id, style_id, default): + instance = self._instances.get(number_id) + if instance is None: + return default + levels = dict(self._abstracts.get(instance.abstract_id, {})) + levels.update(instance.overrides) + for list_level, definition in levels.items(): + if definition.paragraph_style == style_id: + return list_level + return default + + def _resolve_level(self, number_id, list_level): + instance = self._instances.get(number_id) + if instance is None: + return None + return instance.overrides.get(list_level) or self._abstracts.get(instance.abstract_id, {}).get(list_level) + + def _heading_level(self, paragraph): + properties = paragraph._p.pPr + outline_level = _child_int(properties, "w:outlineLvl") + if outline_level is not None and 0 <= outline_level < 9: + return outline_level + 1 + + style = paragraph.style + seen = set() + while style is not None and style.style_id not in seen: + seen.add(style.style_id) + outline_level = _child_int(style.element.pPr, "w:outlineLvl") + if outline_level is not None and 0 <= outline_level < 9: + return outline_level + 1 + match = re.match(r"(?:heading|заголовок)\s*(\d+)$", style.name or "", re.IGNORECASE) + if match: + return int(match.group(1)) + style = style.base_style + return None + + def _render_marker(self, template, number_id, values): + marker = template + for index in range(9): + placeholder = f"%{index + 1}" + if placeholder not in marker: + continue + definition = self._resolve_level(number_id, index) + value = values[index] or (definition.start if definition is not None else 1) + number_format = definition.number_format if definition is not None else "decimal" + marker = marker.replace(placeholder, _format_number(value, number_format)) + return marker + + +def extract_numbered_headings(document): + resolver = DOCXNumberingResolver(document) + headings = [] + for paragraph in document.paragraphs: + heading = resolver.numbered_heading(paragraph) + if heading is not None: + headings.append(heading) + return headings + + +def apply_numbered_headings_to_markdown(markdown, headings): + if not markdown or not headings: + return markdown + lines = markdown.splitlines() + heading_index = 0 + for index, line in enumerate(lines): + match = re.match(r"^(#{1,6})\s+(.*?)\s*$", line) + if not match: + continue + text = re.sub(r"[*_~`]", "", match.group(2)).strip() + for candidate_index in range(heading_index, len(headings)): + heading = headings[candidate_index] + if text != heading.text: + continue + lines[index] = f"{'#' * min(max(heading.level, 1), 6)} {heading.numbered_text}" + heading_index = candidate_index + 1 + break + return "\n".join(lines) + + +def _properties_numbering_reference(properties): + if properties is None or properties.numPr is None or properties.numPr.numId is None: + return None + number_id = _value_int(properties.numPr.numId) + if number_id is None or number_id == 0: + return None + list_level = _value_int(properties.numPr.ilvl) + return number_id, list_level or 0 + + +def _parse_level(element): + return _NumberingLevel( + start=_child_int(element, "w:start") or 1, + number_format=_child_value(element, "w:numFmt") or "decimal", + level_text=_child_value(element, "w:lvlText") or "", + paragraph_style=_child_value(element, "w:pStyle") or "", + ) + + +def _child_value(parent, tag): + if parent is None: + return None + child = parent.find(qn(tag)) + return child.get(qn("w:val")) if child is not None else None + + +def _child_int(parent, tag): + value = _child_value(parent, tag) + try: + return int(value) if value is not None else None + except ValueError: + return None + + +def _int_attribute(element, attribute): + try: + value = element.get(qn(attribute)) + return int(value) if value is not None else None + except ValueError: + return None + + +def _value_int(element): + if element is None: + return None + try: + return int(element.val) + except (TypeError, ValueError): + return None + + +def _format_number(value, number_format): + if number_format == "lowerLetter": + return _format_letters(value, False) + if number_format == "upperLetter": + return _format_letters(value, True) + if number_format == "lowerRoman": + return _format_roman(value).lower() + if number_format == "upperRoman": + return _format_roman(value) + if number_format == "decimalZero" and 0 <= value < 10: + return f"0{value}" + return str(value) + + +def _format_letters(value, upper): + if value <= 0: + return str(value) + result = "" + base = ord("A" if upper else "a") + while value > 0: + value -= 1 + result = chr(base + value % 26) + result + value //= 26 + return result + + +def _format_roman(value): + if value <= 0 or value > 3999: + return str(value) + symbols = ( + (1000, "M"), + (900, "CM"), + (500, "D"), + (400, "CD"), + (100, "C"), + (90, "XC"), + (50, "L"), + (40, "XL"), + (10, "X"), + (9, "IX"), + (5, "V"), + (4, "IV"), + (1, "I"), + ) + result = "" + for number, symbol in symbols: + while value >= number: + result += symbol + value -= number + return result diff --git a/deepdoc/parser/docx_parser.py b/deepdoc/parser/docx_parser.py index 48f8db80648..fc5d50822c4 100644 --- a/deepdoc/parser/docx_parser.py +++ b/deepdoc/parser/docx_parser.py @@ -28,6 +28,7 @@ UnrecognizedImageError, ) from rag.utils.lazy_image import LazyImage +from deepdoc.parser.docx_numbering import DOCXNumberingResolver class RAGFlowDocxParser: @@ -159,12 +160,14 @@ def blockType(b): def __call__(self, fnm, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): self.doc = Document(fnm) if isinstance(fnm, str) else Document(BytesIO(fnm)) + numbering = DOCXNumberingResolver(self.doc) pn = 0 # parsed page secs = [] # parsed contents for p in self.doc.paragraphs: if pn > to_page: break + numbered_heading = numbering.numbered_heading(p) runs_within_single_paragraph = [] # save runs within the range of pages for run in p.runs: if pn > to_page: @@ -176,7 +179,10 @@ def __call__(self, fnm, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): if "lastRenderedPageBreak" in run._element.xml: pn += 1 - secs.append(("".join(runs_within_single_paragraph), p.style.name if hasattr(p.style, "name") else "")) # then concat run.text as part of the paragraph + paragraph_text = "".join(runs_within_single_paragraph) + if paragraph_text and numbered_heading is not None: + paragraph_text = numbered_heading.numbered_text + secs.append((paragraph_text, p.style.name if hasattr(p.style, "name") else "")) # then concat run.text as part of the paragraph tbls = [self.__extract_table_content(tb) for tb in self.doc.tables] return secs, tbls diff --git a/rag/app/naive.py b/rag/app/naive.py index aac579b4f44..3703133af26 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -43,6 +43,7 @@ from rag.utils.file_utils import extract_embed_file, extract_links_from_pdf, extract_links_from_docx, extract_html from deepdoc.parser import DocxParser, EpubParser, ExcelParser, HtmlParser, JsonParser, MarkdownElementExtractor, MarkdownParser, PdfParser, TxtParser from deepdoc.parser.figure_parser import VisionFigureParser, vision_figure_parser_docx_wrapper_naive, vision_figure_parser_pdf_wrapper +from deepdoc.parser.docx_numbering import DOCXNumberingResolver, apply_numbered_headings_to_markdown, extract_numbered_headings from deepdoc.parser.pdf_parser import PlainParser, VisionParser from deepdoc.parser.docling_parser import DoclingParser from deepdoc.parser.tcadp_parser import TCADPParser @@ -559,6 +560,7 @@ def __get_nearest_title(self, table_index, filename): def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): self.doc = Document(filename) if not binary else Document(BytesIO(binary)) + numbering = DOCXNumberingResolver(self.doc) pn = 0 lines = [] last_image = None @@ -576,9 +578,10 @@ def flush_last_image(): if block.tag.endswith("p"): p = Paragraph(block, self.doc) + numbered_heading = numbering.numbered_heading(p) if from_page <= pn < to_page: - text = p.text.strip() + text = numbered_heading.numbered_text if numbered_heading is not None else p.text.strip() style_name = p.style.name if p.style else "" if text: @@ -683,6 +686,8 @@ def to_markdown(self, filename=None, binary=None, inline_images: bool = True): import mammoth from markdownify import markdownify + document = Document(filename) if binary is None else Document(BytesIO(binary)) + numbered_headings = extract_numbered_headings(document) docx_file = BytesIO(binary) if binary else open(filename, "rb") def _convert_image_to_base64(image): @@ -709,7 +714,7 @@ def _convert_image_to_base64(image): html = result.value markdown_text = markdownify(html) - return markdown_text + return apply_numbered_headings_to_markdown(markdown_text, numbered_headings) finally: if not binary: diff --git a/rag/flow/parser/utils.py b/rag/flow/parser/utils.py index db9eef0c249..397b6882e23 100644 --- a/rag/flow/parser/utils.py +++ b/rag/flow/parser/utils.py @@ -25,6 +25,7 @@ ) from common.constants import LLMType from deepdoc.parser.figure_parser import VisionFigureParser +from deepdoc.parser.docx_numbering import DOCXNumberingResolver from rag.nlp import is_english, random_choices, remove_contents_table @@ -76,16 +77,17 @@ def remove_header_footer_html_blob(blob): def extract_word_outlines(filename, binary=None): doc = Document(filename) if binary is None else Document(BytesIO(binary)) + numbering = DOCXNumberingResolver(doc) outlines = [] for paragraph in doc.paragraphs: - text = paragraph.text.strip() + numbered_heading = numbering.numbered_heading(paragraph) + text = numbered_heading.numbered_text if numbered_heading is not None else paragraph.text.strip() if not text: continue - style_name = paragraph.style.name if paragraph.style else "" - match = re.search(r"Heading\s*(\d+)", style_name, re.I) - if not match: + level = numbered_heading.level if numbered_heading is not None else numbering.heading_level(paragraph) + if level is None: continue - outlines.append((text, int(match.group(1)) - 1, None)) + outlines.append((text, level - 1, None)) return outlines diff --git a/test/unit_test/deepdoc/parser/test_docx_numbering.py b/test/unit_test/deepdoc/parser/test_docx_numbering.py new file mode 100644 index 00000000000..5a658130c8f --- /dev/null +++ b/test/unit_test/deepdoc/parser/test_docx_numbering.py @@ -0,0 +1,96 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from io import BytesIO + +from docx import Document +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + +from deepdoc.parser.docx_numbering import apply_numbered_headings_to_markdown, extract_numbered_headings + + +def _element(tag, value=None, **attributes): + element = OxmlElement(tag) + if value is not None: + element.set(qn("w:val"), str(value)) + for name, attribute_value in attributes.items(): + element.set(qn(f"w:{name}"), str(attribute_value)) + return element + + +def _add_numbering_level(abstract, level, style_id, level_text): + definition = _element("w:lvl", ilvl=level) + definition.append(_element("w:start", 1)) + definition.append(_element("w:numFmt", "decimal")) + definition.append(_element("w:pStyle", style_id)) + definition.append(_element("w:lvlText", level_text)) + abstract.append(definition) + + +def _set_style_numbering(style, number_id, level): + properties = style.element.get_or_add_pPr() + for existing in properties.findall(qn("w:numPr")): + properties.remove(existing) + numbering = _element("w:numPr") + numbering.append(_element("w:ilvl", level)) + numbering.append(_element("w:numId", number_id)) + properties.append(numbering) + + +def _numbered_document(): + document = Document() + numbering_root = document.part.numbering_part.element + abstract_id = 4242 + number_id = 4242 + + abstract = _element("w:abstractNum", abstractNumId=abstract_id) + _add_numbering_level(abstract, 0, "Heading1", "%1") + _add_numbering_level(abstract, 1, "Heading2", "%1.%2") + numbering_root.append(abstract) + + number = _element("w:num", numId=number_id) + number.append(_element("w:abstractNumId", abstract_id)) + numbering_root.append(number) + + _set_style_numbering(document.styles["Heading 1"], number_id, 0) + _set_style_numbering(document.styles["Heading 2"], number_id, 1) + document.add_paragraph("Введение", style="Heading 1") + document.add_paragraph("Порядок работы", style="Heading 1") + document.add_paragraph("Начало работы", style="Heading 2") + + stream = BytesIO() + document.save(stream) + stream.seek(0) + return Document(stream) + + +def test_extract_numbered_headings_from_style_numbering(): + headings = extract_numbered_headings(_numbered_document()) + + assert [(heading.numbered_text, heading.level) for heading in headings] == [ + ("1 Введение", 1), + ("2 Порядок работы", 1), + ("2.1 Начало работы", 2), + ] + + +def test_apply_numbered_headings_to_markdown(): + headings = extract_numbered_headings(_numbered_document()) + + markdown = "# Введение\n\n# Порядок работы\n\n## Начало работы\n" + + assert apply_numbered_headings_to_markdown(markdown, headings) == "# 1 Введение\n\n# 2 Порядок работы\n\n## 2.1 Начало работы" From 69307b96fa7dad176a987e7fca4f0ae4bc7c1f85 Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 12:59:25 +0300 Subject: [PATCH 3/9] fix: restore DOCX numbering in setext headings --- deepdoc/parser/docx_numbering.py | 10 +++++++--- test/unit_test/deepdoc/parser/test_docx_numbering.py | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/deepdoc/parser/docx_numbering.py b/deepdoc/parser/docx_numbering.py index 60ea1d7e9da..a063b0ae6bb 100644 --- a/deepdoc/parser/docx_numbering.py +++ b/deepdoc/parser/docx_numbering.py @@ -217,14 +217,18 @@ def apply_numbered_headings_to_markdown(markdown, headings): heading_index = 0 for index, line in enumerate(lines): match = re.match(r"^(#{1,6})\s+(.*?)\s*$", line) - if not match: + is_setext_heading = index + 1 < len(lines) and re.fullmatch(r"\s*[=-]+\s*", lines[index + 1]) is not None + if match is None and not is_setext_heading: continue - text = re.sub(r"[*_~`]", "", match.group(2)).strip() + text = re.sub(r"[*_~`]", "", match.group(2) if match is not None else line).strip() for candidate_index in range(heading_index, len(headings)): heading = headings[candidate_index] if text != heading.text: continue - lines[index] = f"{'#' * min(max(heading.level, 1), 6)} {heading.numbered_text}" + if match is not None: + lines[index] = f"{'#' * min(max(heading.level, 1), 6)} {heading.numbered_text}" + else: + lines[index] = heading.numbered_text heading_index = candidate_index + 1 break return "\n".join(lines) diff --git a/test/unit_test/deepdoc/parser/test_docx_numbering.py b/test/unit_test/deepdoc/parser/test_docx_numbering.py index 5a658130c8f..15a6f40d25c 100644 --- a/test/unit_test/deepdoc/parser/test_docx_numbering.py +++ b/test/unit_test/deepdoc/parser/test_docx_numbering.py @@ -94,3 +94,11 @@ def test_apply_numbered_headings_to_markdown(): markdown = "# Введение\n\n# Порядок работы\n\n## Начало работы\n" assert apply_numbered_headings_to_markdown(markdown, headings) == "# 1 Введение\n\n# 2 Порядок работы\n\n## 2.1 Начало работы" + + +def test_apply_numbered_headings_to_setext_markdown(): + headings = extract_numbered_headings(_numbered_document()) + + markdown = "Введение\n========\n\nПорядок работы\n===============\n\nНачало работы\n-------------\n" + + assert apply_numbered_headings_to_markdown(markdown, headings) == "1 Введение\n========\n\n2 Порядок работы\n===============\n\n2.1 Начало работы\n-------------" From d94e399251206c6ba873a8993388c9f0dc1eb5cc Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 13:09:39 +0300 Subject: [PATCH 4/9] fix: respect disabled DOCX heading numbering --- deepdoc/parser/docx_numbering.py | 14 +++++++- internal/parser/parser/docx_numbering.go | 27 +++++++++++---- internal/parser/parser/docx_numbering_test.go | 3 ++ .../deepdoc/parser/test_docx_numbering.py | 33 +++++++++++++++---- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/deepdoc/parser/docx_numbering.py b/deepdoc/parser/docx_numbering.py index a063b0ae6bb..04eb783e440 100644 --- a/deepdoc/parser/docx_numbering.py +++ b/deepdoc/parser/docx_numbering.py @@ -128,7 +128,11 @@ def _parse_numbering(self, document): self._instance_order.append(number_id) def _paragraph_numbering_reference(self, paragraph): - reference = _properties_numbering_reference(paragraph._p.pPr) + properties = paragraph._p.pPr + if _is_numbering_disabled(properties): + return None + + reference = _properties_numbering_reference(properties) if reference is not None: return reference @@ -136,6 +140,8 @@ def _paragraph_numbering_reference(self, paragraph): seen = set() while style is not None and style.style_id not in seen: seen.add(style.style_id) + if _is_numbering_disabled(style.element.pPr): + return None reference = _properties_numbering_reference(style.element.pPr) if reference is not None: number_id, list_level = reference @@ -244,6 +250,12 @@ def _properties_numbering_reference(properties): return number_id, list_level or 0 +def _is_numbering_disabled(properties): + if properties is None or properties.numPr is None or properties.numPr.numId is None: + return False + return _value_int(properties.numPr.numId) == 0 + + def _parse_level(element): return _NumberingLevel( start=_child_int(element, "w:start") or 1, diff --git a/internal/parser/parser/docx_numbering.go b/internal/parser/parser/docx_numbering.go index 1eea6c8ed8b..9372e58a0c1 100644 --- a/internal/parser/parser/docx_numbering.go +++ b/internal/parser/parser/docx_numbering.go @@ -116,9 +116,10 @@ type docxParagraphNumbering struct { } type docxResolvedStyle struct { - Name string - NumberingRef *docxParagraphNumbering - OutlineLevel *int + Name string + NumberingRef *docxParagraphNumbering + NumberingDisabled bool + OutlineLevel *int } // extractDOCXNumberedHeadings materializes Word's display-only heading @@ -146,10 +147,12 @@ func extractDOCXNumberedHeadings(data []byte) []docxNumberedHeading { } style := resolveDOCXStyle(styleID, styles, nil) ref := paragraphNumberingRef(paragraph.Properties) - if ref == nil { + numberingDisabled := isParagraphNumberingDisabled(paragraph.Properties) + if ref == nil && !numberingDisabled { ref = style.NumberingRef + numberingDisabled = style.NumberingDisabled } - if ref == nil { + if ref == nil && !numberingDisabled { ref = numberingRefForParagraphStyle(styleID, numbering) } if ref == nil { @@ -337,6 +340,14 @@ func paragraphNumberingRef(properties *docxXMLParagraphProperties) *docxParagrap return &docxParagraphNumbering{ID: id, Level: level} } +func isParagraphNumberingDisabled(properties *docxXMLParagraphProperties) bool { + if properties == nil || properties.NumberingRef == nil { + return false + } + id, ok := parseDOCXXMLInt(properties.NumberingRef.ID) + return ok && id == 0 +} + func resolveDOCXStyle(id string, styles map[string]docxXMLStyle, seen map[string]bool) docxResolvedStyle { if id == "" { return docxResolvedStyle{} @@ -359,8 +370,12 @@ func resolveDOCXStyle(id string, styles map[string]docxXMLStyle, seen map[string if style.Name != nil { resolved.Name = style.Name.Value } - if ref := paragraphNumberingRef(style.Properties); ref != nil { + if isParagraphNumberingDisabled(style.Properties) { + resolved.NumberingRef = nil + resolved.NumberingDisabled = true + } else if ref := paragraphNumberingRef(style.Properties); ref != nil { resolved.NumberingRef = ref + resolved.NumberingDisabled = false } if style.Properties != nil { if level, ok := parseDOCXXMLInt(style.Properties.OutlineLevel); ok { diff --git a/internal/parser/parser/docx_numbering_test.go b/internal/parser/parser/docx_numbering_test.go index d7b1fee4a21..72c75d8274f 100644 --- a/internal/parser/parser/docx_numbering_test.go +++ b/internal/parser/parser/docx_numbering_test.go @@ -12,6 +12,8 @@ func numberedHeadingsDOCX(t *testing.T) []byte { parts := map[string]string{ "word/document.xml": ` + Unnumbered by style + Unnumbered introduction Общие сведения Установка Начало работы @@ -19,6 +21,7 @@ func numberedHeadingsDOCX(t *testing.T) []byte { `, "word/styles.xml": ` + `, diff --git a/test/unit_test/deepdoc/parser/test_docx_numbering.py b/test/unit_test/deepdoc/parser/test_docx_numbering.py index 15a6f40d25c..a254aaaa137 100644 --- a/test/unit_test/deepdoc/parser/test_docx_numbering.py +++ b/test/unit_test/deepdoc/parser/test_docx_numbering.py @@ -17,6 +17,7 @@ from io import BytesIO from docx import Document +from docx.enum.style import WD_STYLE_TYPE from docx.oxml import OxmlElement from docx.oxml.ns import qn @@ -51,6 +52,14 @@ def _set_style_numbering(style, number_id, level): properties.append(numbering) +def _disable_paragraph_numbering(paragraph): + properties = paragraph._p.get_or_add_pPr() + numbering = _element("w:numPr") + numbering.append(_element("w:ilvl", 0)) + numbering.append(_element("w:numId", 0)) + properties.append(numbering) + + def _numbered_document(): document = Document() numbering_root = document.part.numbering_part.element @@ -68,7 +77,13 @@ def _numbered_document(): _set_style_numbering(document.styles["Heading 1"], number_id, 0) _set_style_numbering(document.styles["Heading 2"], number_id, 1) - document.add_paragraph("Введение", style="Heading 1") + unnumbered_heading = document.styles.add_style("Unnumbered Heading", WD_STYLE_TYPE.PARAGRAPH) + unnumbered_heading.base_style = document.styles["Heading 1"] + _set_style_numbering(unnumbered_heading, 0, 0) + document.add_paragraph("Аннотация", style=unnumbered_heading) + introduction = document.add_paragraph("Введение", style="Heading 1") + _disable_paragraph_numbering(introduction) + document.add_paragraph("Общие сведения", style="Heading 1") document.add_paragraph("Порядок работы", style="Heading 1") document.add_paragraph("Начало работы", style="Heading 2") @@ -82,23 +97,29 @@ def test_extract_numbered_headings_from_style_numbering(): headings = extract_numbered_headings(_numbered_document()) assert [(heading.numbered_text, heading.level) for heading in headings] == [ - ("1 Введение", 1), + ("1 Общие сведения", 1), ("2 Порядок работы", 1), ("2.1 Начало работы", 2), ] +def test_explicit_numbering_disable_wins_over_numbered_style(): + headings = extract_numbered_headings(_numbered_document()) + + assert not any(heading.text in {"Аннотация", "Введение"} for heading in headings) + + def test_apply_numbered_headings_to_markdown(): headings = extract_numbered_headings(_numbered_document()) - markdown = "# Введение\n\n# Порядок работы\n\n## Начало работы\n" + markdown = "# Введение\n\n# Общие сведения\n\n# Порядок работы\n\n## Начало работы\n" - assert apply_numbered_headings_to_markdown(markdown, headings) == "# 1 Введение\n\n# 2 Порядок работы\n\n## 2.1 Начало работы" + assert apply_numbered_headings_to_markdown(markdown, headings) == "# Введение\n\n# 1 Общие сведения\n\n# 2 Порядок работы\n\n## 2.1 Начало работы" def test_apply_numbered_headings_to_setext_markdown(): headings = extract_numbered_headings(_numbered_document()) - markdown = "Введение\n========\n\nПорядок работы\n===============\n\nНачало работы\n-------------\n" + markdown = "Введение\n========\n\nОбщие сведения\n===============\n\nПорядок работы\n===============\n\nНачало работы\n-------------\n" - assert apply_numbered_headings_to_markdown(markdown, headings) == "1 Введение\n========\n\n2 Порядок работы\n===============\n\n2.1 Начало работы\n-------------" + assert apply_numbered_headings_to_markdown(markdown, headings) == "Введение\n========\n\n1 Общие сведения\n===============\n\n2 Порядок работы\n===============\n\n2.1 Начало работы\n-------------" From da3cbe40b89d480819b4b23438786971da5f4a24 Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 13:50:59 +0300 Subject: [PATCH 5/9] feat: make DOCX automatic numbering configurable --- deepdoc/parser/docx_numbering.py | 12 ++++++---- deepdoc/parser/docx_parser.py | 4 ++-- internal/ingestion/component/parser.go | 11 +++++---- .../component/parser_dispatch_test.go | 4 ++++ internal/parser/parser/docx_numbering.go | 9 ++++++++ internal/parser/parser/docx_numbering_test.go | 7 ++++++ internal/parser/parser/docx_parser.go | 16 ++++++++----- .../parser/parser/docx_parser_cgo_test.go | 12 ++++++++++ rag/app/naive.py | 8 +++---- rag/flow/parser/parser.py | 8 ++++--- rag/flow/parser/utils.py | 4 ++-- .../deepdoc/parser/test_docx_numbering.py | 6 +++++ web/src/locales/en.ts | 3 +++ web/src/locales/ru.ts | 3 +++ web/src/pages/agent/constant/pipeline.tsx | 1 + .../form/parser-form/common-form-fields.tsx | 23 +++++++++++++++++++ .../pages/agent/form/parser-form/index.tsx | 2 ++ .../form/parser-form/word-form-fields.tsx | 10 +++++++- web/src/pages/agent/utils.ts | 2 ++ 19 files changed, 118 insertions(+), 27 deletions(-) diff --git a/deepdoc/parser/docx_numbering.py b/deepdoc/parser/docx_numbering.py index 04eb783e440..749ca3328a0 100644 --- a/deepdoc/parser/docx_numbering.py +++ b/deepdoc/parser/docx_numbering.py @@ -44,15 +44,19 @@ class _NumberingInstance: class DOCXNumberingResolver: """Materialize display-only Word numbering while paragraphs are visited.""" - def __init__(self, document): + def __init__(self, document, enabled=True): + self._enabled = enabled self._abstracts: dict[int, dict[int, _NumberingLevel]] = {} self._instances: dict[int, _NumberingInstance] = {} self._instance_order: list[int] = [] self._counters: dict[int, list[int]] = {} - self._parse_numbering(document) + if enabled: + self._parse_numbering(document) def numbered_heading(self, paragraph) -> NumberedHeading | None: """Advance numbering state and return a materialized heading, if any.""" + if not self._enabled: + return None reference = self._paragraph_numbering_reference(paragraph) if reference is None: return None @@ -206,8 +210,8 @@ def _render_marker(self, template, number_id, values): return marker -def extract_numbered_headings(document): - resolver = DOCXNumberingResolver(document) +def extract_numbered_headings(document, enabled=True): + resolver = DOCXNumberingResolver(document, enabled=enabled) headings = [] for paragraph in document.paragraphs: heading = resolver.numbered_heading(paragraph) diff --git a/deepdoc/parser/docx_parser.py b/deepdoc/parser/docx_parser.py index fc5d50822c4..88be7de5b56 100644 --- a/deepdoc/parser/docx_parser.py +++ b/deepdoc/parser/docx_parser.py @@ -158,9 +158,9 @@ def blockType(b): return lines return ["\n".join(lines)] - def __call__(self, fnm, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): + def __call__(self, fnm, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, extract_automatic_numbering=True): self.doc = Document(fnm) if isinstance(fnm, str) else Document(BytesIO(fnm)) - numbering = DOCXNumberingResolver(self.doc) + numbering = DOCXNumberingResolver(self.doc, enabled=extract_automatic_numbering) pn = 0 # parsed page secs = [] # parsed contents for p in self.doc.paragraphs: diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go index 5031a5b4bed..cba114ab700 100644 --- a/internal/ingestion/component/parser.go +++ b/internal/ingestion/component/parser.go @@ -262,11 +262,12 @@ func defaultSetups() map[string]schema.ParserSetup { "output_format": "json", }, "docx": { - "flatten_media_to_text": false, - "remove_toc": false, - "remove_header_footer": false, - "suffix": []string{"docx"}, - "output_format": "json", + "flatten_media_to_text": false, + "extract_automatic_numbering": true, + "remove_toc": false, + "remove_header_footer": false, + "suffix": []string{"docx"}, + "output_format": "json", }, "markdown": { "flatten_media_to_text": false, diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go index 99fd7cb764a..0b13bcdf886 100644 --- a/internal/ingestion/component/parser_dispatch_test.go +++ b/internal/ingestion/component/parser_dispatch_test.go @@ -303,6 +303,10 @@ func TestDefaultSetups_DOCX_OutputFormatMarkdown(t *testing.T) { if got != "json" { t.Errorf("docx.output_format = %q, want %q", got, "json") } + extractNumbering, ok := docx["extract_automatic_numbering"].(bool) + if !ok || !extractNumbering { + t.Errorf("docx.extract_automatic_numbering = %v, want true", docx["extract_automatic_numbering"]) + } } func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) { diff --git a/internal/parser/parser/docx_numbering.go b/internal/parser/parser/docx_numbering.go index 9372e58a0c1..a8bd94bf40d 100644 --- a/internal/parser/parser/docx_numbering.go +++ b/internal/parser/parser/docx_numbering.go @@ -122,6 +122,15 @@ type docxResolvedStyle struct { OutlineLevel *int } +// extractDOCXNumberedHeadingsIfEnabled avoids parsing numbering definitions +// when automatic numbering extraction is disabled in the DOCX setup. +func extractDOCXNumberedHeadingsIfEnabled(data []byte, enabled bool) []docxNumberedHeading { + if !enabled { + return nil + } + return extractDOCXNumberedHeadings(data) +} + // extractDOCXNumberedHeadings materializes Word's display-only heading // numbers. Word stores paragraph text in document.xml and the visible marker // separately in numbering.xml, so office_oxide's text runs cannot contain it. diff --git a/internal/parser/parser/docx_numbering_test.go b/internal/parser/parser/docx_numbering_test.go index 72c75d8274f..eb59a1375bd 100644 --- a/internal/parser/parser/docx_numbering_test.go +++ b/internal/parser/parser/docx_numbering_test.go @@ -64,6 +64,13 @@ func TestExtractDOCXNumberedHeadings(t *testing.T) { } } +func TestExtractDOCXNumberedHeadingsDisabled(t *testing.T) { + got := extractDOCXNumberedHeadingsIfEnabled(numberedHeadingsDOCX(t), false) + if got != nil { + t.Fatalf("disabled extraction returned %#v, want nil", got) + } +} + func TestApplyDOCXNumberedHeadingsToSections(t *testing.T) { headings := extractDOCXNumberedHeadings(numberedHeadingsDOCX(t)) items := []map[string]any{ diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go index f0a8f64082b..c5a7e72dbc6 100644 --- a/internal/parser/parser/docx_parser.go +++ b/internal/parser/parser/docx_parser.go @@ -32,14 +32,15 @@ import ( // so they compile and test without native libraries. The !cgo build // provides a stub DOCXParser in office_parsers_no_cgo.go. type DOCXParser struct { - libType string - outputFormat string // from DSL config; "json" or "markdown" - RemoveTOC bool - RemoveHeaderFooter bool + libType string + outputFormat string // from DSL config; "json" or "markdown" + extractAutomaticNumbering bool + RemoveTOC bool + RemoveHeaderFooter bool } func NewDOCXParser() *DOCXParser { - return &DOCXParser{} + return &DOCXParser{extractAutomaticNumbering: true} } // ConfigureFromSetup implements parserSetupConfigurer, receiving the @@ -58,6 +59,9 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) { if v, ok := setup["remove_header_footer"].(bool); ok { p.RemoveHeaderFooter = v } + if v, ok := setup["extract_automatic_numbering"].(bool); ok { + p.extractAutomaticNumbering = v + } } // ParseWithResult produces structured JSON items (when @@ -82,7 +86,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data // Extract IR JSON for section building (JSON path) and // embedded-image extraction (both paths). irJSON, irErr := doc.ToIRJSON() - numberedHeadings := extractDOCXNumberedHeadings(data) + numberedHeadings := extractDOCXNumberedHeadingsIfEnabled(data, p.extractAutomaticNumbering) var figures []DOCXFigure if irErr == nil { figures = extractDOCXFiguresFromIR(irJSON) diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go index 65ff846f7f6..d2f013c9395 100644 --- a/internal/parser/parser/docx_parser_cgo_test.go +++ b/internal/parser/parser/docx_parser_cgo_test.go @@ -57,6 +57,18 @@ func TestDOCXParser_ConfigureFromSetup_JSON(t *testing.T) { } } +func TestDOCXParser_ConfigureAutomaticNumbering(t *testing.T) { + p := NewDOCXParser() + if !p.extractAutomaticNumbering { + t.Fatal("automatic numbering must be enabled by default") + } + + p.ConfigureFromSetup(map[string]any{"extract_automatic_numbering": false}) + if p.extractAutomaticNumbering { + t.Fatal("automatic numbering was not disabled from setup") + } +} + func TestDOCXParser_ConfigureFromSetup_Markdown(t *testing.T) { ctx := t.Context() p := NewDOCXParser() diff --git a/rag/app/naive.py b/rag/app/naive.py index 3703133af26..598dc6898c3 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -558,9 +558,9 @@ def __get_nearest_title(self, table_index, filename): return "" - def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): + def __call__(self, filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, extract_automatic_numbering=True): self.doc = Document(filename) if not binary else Document(BytesIO(binary)) - numbering = DOCXNumberingResolver(self.doc) + numbering = DOCXNumberingResolver(self.doc, enabled=extract_automatic_numbering) pn = 0 lines = [] last_image = None @@ -675,7 +675,7 @@ def flush_last_image(): return new_line - def to_markdown(self, filename=None, binary=None, inline_images: bool = True): + def to_markdown(self, filename=None, binary=None, inline_images: bool = True, extract_automatic_numbering=True): """ This function uses mammoth, licensed under the BSD 2-Clause License. """ @@ -687,7 +687,7 @@ def to_markdown(self, filename=None, binary=None, inline_images: bool = True): from markdownify import markdownify document = Document(filename) if binary is None else Document(BytesIO(binary)) - numbered_headings = extract_numbered_headings(document) + numbered_headings = extract_numbered_headings(document, enabled=extract_automatic_numbering) docx_file = BytesIO(binary) if binary else open(filename, "rb") def _convert_image_to_base64(image): diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py index 4e807458133..b00b3bff25f 100644 --- a/rag/flow/parser/parser.py +++ b/rag/flow/parser/parser.py @@ -153,6 +153,7 @@ def __init__(self): }, "docx": { "flatten_media_to_text": False, + "extract_automatic_numbering": True, "remove_toc": False, "remove_header_footer": False, "suffix": [ @@ -943,14 +944,15 @@ def _docx(self, name, blob, **kwargs): return docx_parser = Docx() + extract_automatic_numbering = conf.get("extract_automatic_numbering", True) # Extract heading-based outlines for metadata and TOC removal. - outlines = extract_word_outlines(name, blob) + outlines = extract_word_outlines(name, blob, extract_automatic_numbering=extract_automatic_numbering) self.set_output("file", {**kwargs.get("file", {}), "outlines": outlines}) # JSON output keeps text/image blocks and appends table HTML as table items. if conf.get("output_format") == "json": - main_sections = docx_parser(name, binary=blob) + main_sections = docx_parser(name, binary=blob, extract_automatic_numbering=extract_automatic_numbering) if conf.get("remove_header_footer"): header_footer_texts = extract_docx_header_footer_texts(binary=blob) main_sections = remove_header_footer_docx_sections(main_sections, header_footer_texts) @@ -985,7 +987,7 @@ def _docx(self, name, blob, **kwargs): # Markdown output removes TOC on plain markdown lines before writing back. elif conf.get("output_format") == "markdown": - markdown_text = docx_parser.to_markdown(name, binary=blob) + markdown_text = docx_parser.to_markdown(name, binary=blob, extract_automatic_numbering=extract_automatic_numbering) if conf.get("remove_header_footer"): header_footer_texts = extract_docx_header_footer_texts(binary=blob) markdown_lines = remove_header_footer_docx_sections(markdown_text.split("\n"), header_footer_texts) diff --git a/rag/flow/parser/utils.py b/rag/flow/parser/utils.py index 397b6882e23..19e94b88470 100644 --- a/rag/flow/parser/utils.py +++ b/rag/flow/parser/utils.py @@ -75,9 +75,9 @@ def remove_header_footer_html_blob(blob): return str(soup).encode("utf-8") -def extract_word_outlines(filename, binary=None): +def extract_word_outlines(filename, binary=None, extract_automatic_numbering=True): doc = Document(filename) if binary is None else Document(BytesIO(binary)) - numbering = DOCXNumberingResolver(doc) + numbering = DOCXNumberingResolver(doc, enabled=extract_automatic_numbering) outlines = [] for paragraph in doc.paragraphs: numbered_heading = numbering.numbered_heading(paragraph) diff --git a/test/unit_test/deepdoc/parser/test_docx_numbering.py b/test/unit_test/deepdoc/parser/test_docx_numbering.py index a254aaaa137..fcfae62bfc2 100644 --- a/test/unit_test/deepdoc/parser/test_docx_numbering.py +++ b/test/unit_test/deepdoc/parser/test_docx_numbering.py @@ -109,6 +109,12 @@ def test_explicit_numbering_disable_wins_over_numbered_style(): assert not any(heading.text in {"Аннотация", "Введение"} for heading in headings) +def test_numbering_extraction_can_be_disabled(): + headings = extract_numbered_headings(_numbered_document(), enabled=False) + + assert headings == [] + + def test_apply_numbered_headings_to_markdown(): headings = extract_numbered_headings(_numbered_document()) diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index c55be3aa527..7a8ee06df5d 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -2520,6 +2520,9 @@ Best for: Documents with flowing, contextually connected content — such as boo removeTocTip: 'Remove the table of contents included in the original PDF, so it is not parsed as regular content or chunked for retrieval.', removeHeaderFooter: 'Remove header and footer', + extractAutomaticNumbering: 'Extract automatic numbering', + extractAutomaticNumberingTip: + 'Add numbering generated by Word to numbered DOCX headings.', autoPlay: 'Auto play audio', downloadFileTypeTip: 'The file type to download', downloadFileType: 'Download file type', diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.ts index 0288c270c4e..ae12ca7b4b7 100644 --- a/web/src/locales/ru.ts +++ b/web/src/locales/ru.ts @@ -1566,6 +1566,9 @@ export default { flowNum: 'N', test: 'Тест', extractDepth: 'Глубина извлечения', + extractAutomaticNumbering: 'Извлекать автоматическую нумерацию', + extractAutomaticNumberingTip: + 'Добавлять к заголовкам DOCX номера, автоматически созданные в Word.', format: 'Формат', basic: 'базовый', advanced: 'продвинутый', diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx index 78032c1c2d3..035a96aaf30 100644 --- a/web/src/pages/agent/constant/pipeline.tsx +++ b/web/src/pages/agent/constant/pipeline.tsx @@ -247,6 +247,7 @@ export const initialParserValues = { output_format: DocxOutputFormat.Json, preprocess: PreprocessValue.main_content, flatten_media_to_text: false, + extract_automatic_numbering: true, remove_header_footer: false, }, { diff --git a/web/src/pages/agent/form/parser-form/common-form-fields.tsx b/web/src/pages/agent/form/parser-form/common-form-fields.tsx index 2d0a1fd1453..8c908b35f5b 100644 --- a/web/src/pages/agent/form/parser-form/common-form-fields.tsx +++ b/web/src/pages/agent/form/parser-form/common-form-fields.tsx @@ -165,6 +165,29 @@ export function RemoveHeaderFooterFormField({ prefix }: CommonProps) { ); } +export function ExtractAutomaticNumberingFormField({ prefix }: CommonProps) { + const { t } = useTranslation(); + return ( + + {(field) => ( + { + field.onChange?.(checked); + }} + /> + )} + + ); +} + export function LanguageFormField({ prefix }: CommonProps) { const { t } = useTranslation(); const crossLanguageOptions = useCrossLanguageOptions(); diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsx index c94170a89be..5937ba3cf6d 100644 --- a/web/src/pages/agent/form/parser-form/index.tsx +++ b/web/src/pages/agent/form/parser-form/index.tsx @@ -86,6 +86,7 @@ export const FormSchema = z.object({ enable_multi_column: z.boolean().optional(), remove_toc: z.boolean().optional(), remove_header_footer: z.boolean().optional(), + extract_automatic_numbering: z.boolean().optional(), pages: z .array(z.object({ from: z.coerce.number(), to: z.coerce.number() })) .optional(), @@ -216,6 +217,7 @@ const ParserForm = ({ table_result_type: '', markdown_image_response_type: '', remove_header_footer: false, + extract_automatic_numbering: true, // preprocess: [], }); }, [append]); diff --git a/web/src/pages/agent/form/parser-form/word-form-fields.tsx b/web/src/pages/agent/form/parser-form/word-form-fields.tsx index 77ce5314faa..8c188f7535b 100644 --- a/web/src/pages/agent/form/parser-form/word-form-fields.tsx +++ b/web/src/pages/agent/form/parser-form/word-form-fields.tsx @@ -4,8 +4,10 @@ import { } from '@/components/model-tree-select'; import { useWatch } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; +import { FileType } from '../../constant/pipeline'; import { useOwnerTenantId } from '../../context'; import { + ExtractAutomaticNumberingFormField, FlattenMediaToTextFormField, OutputFormatFormFieldProps, RemoveHeaderFooterFormField, @@ -13,7 +15,10 @@ import { } from './common-form-fields'; import { buildFieldNameWithPrefix } from './utils'; -export function WordFormFields({ prefix }: OutputFormatFormFieldProps) { +export function WordFormFields({ + prefix, + fileType, +}: OutputFormatFormFieldProps) { const { t } = useTranslation(); const ownerTenantId = useOwnerTenantId(); const flattenMediaToText = useWatch({ @@ -24,6 +29,9 @@ export function WordFormFields({ prefix }: OutputFormatFormFieldProps) { <> + {fileType === FileType.Docx && ( + + )} {!flattenMediaToText && ( Date: Thu, 13 Aug 2026 13:52:06 +0300 Subject: [PATCH 6/9] fix: scope automatic numbering control to DOCX --- web/src/locales/en.ts | 2 +- web/src/locales/ru.ts | 2 +- .../form/parser-form/common-form-fields.tsx | 23 ------------------- .../form/parser-form/word-form-fields.tsx | 19 +++++++++++++-- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 7a8ee06df5d..8e61809caa5 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -2522,7 +2522,7 @@ Best for: Documents with flowing, contextually connected content — such as boo removeHeaderFooter: 'Remove header and footer', extractAutomaticNumbering: 'Extract automatic numbering', extractAutomaticNumberingTip: - 'Add numbering generated by Word to numbered DOCX headings.', + 'Extract numbers from automatically numbered headings.', autoPlay: 'Auto play audio', downloadFileTypeTip: 'The file type to download', downloadFileType: 'Download file type', diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.ts index ae12ca7b4b7..1b45e93b246 100644 --- a/web/src/locales/ru.ts +++ b/web/src/locales/ru.ts @@ -1568,7 +1568,7 @@ export default { extractDepth: 'Глубина извлечения', extractAutomaticNumbering: 'Извлекать автоматическую нумерацию', extractAutomaticNumberingTip: - 'Добавлять к заголовкам DOCX номера, автоматически созданные в Word.', + 'Извлекать номера автоматически пронумерованных заголовков.', format: 'Формат', basic: 'базовый', advanced: 'продвинутый', diff --git a/web/src/pages/agent/form/parser-form/common-form-fields.tsx b/web/src/pages/agent/form/parser-form/common-form-fields.tsx index 8c908b35f5b..2d0a1fd1453 100644 --- a/web/src/pages/agent/form/parser-form/common-form-fields.tsx +++ b/web/src/pages/agent/form/parser-form/common-form-fields.tsx @@ -165,29 +165,6 @@ export function RemoveHeaderFooterFormField({ prefix }: CommonProps) { ); } -export function ExtractAutomaticNumberingFormField({ prefix }: CommonProps) { - const { t } = useTranslation(); - return ( - - {(field) => ( - { - field.onChange?.(checked); - }} - /> - )} - - ); -} - export function LanguageFormField({ prefix }: CommonProps) { const { t } = useTranslation(); const crossLanguageOptions = useCrossLanguageOptions(); diff --git a/web/src/pages/agent/form/parser-form/word-form-fields.tsx b/web/src/pages/agent/form/parser-form/word-form-fields.tsx index 8c188f7535b..c1b2961a8d2 100644 --- a/web/src/pages/agent/form/parser-form/word-form-fields.tsx +++ b/web/src/pages/agent/form/parser-form/word-form-fields.tsx @@ -2,12 +2,13 @@ import { ModelTreeSelectFormField, ModelTypeMap, } from '@/components/model-tree-select'; +import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { Switch } from '@/components/ui/switch'; import { useWatch } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { FileType } from '../../constant/pipeline'; import { useOwnerTenantId } from '../../context'; import { - ExtractAutomaticNumberingFormField, FlattenMediaToTextFormField, OutputFormatFormFieldProps, RemoveHeaderFooterFormField, @@ -30,7 +31,21 @@ export function WordFormFields({ {fileType === FileType.Docx && ( - + + {(field) => ( + + )} + )} {!flattenMediaToText && ( From b5a3294a4c3326e161cd74e42799bb9c0db85fa0 Mon Sep 17 00:00:00 2001 From: ilarionov_n Date: Thu, 13 Aug 2026 13:55:37 +0300 Subject: [PATCH 7/9] fix: translate parser and title chunker settings --- web/src/locales/ru.ts | 54 ++++++++++++++----- .../pages/agent/form/parser-form/index.tsx | 2 +- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.ts index 1b45e93b246..3979a0280bf 100644 --- a/web/src/locales/ru.ts +++ b/web/src/locales/ru.ts @@ -60,6 +60,7 @@ export default { previousPage: 'Назад', nextPage: 'Вперед', add: 'Добавить', + automatic: 'Автоматически', remove: 'Удалить', search: 'Поиск', noDataFound: 'Данные не найдены.', @@ -1566,9 +1567,6 @@ export default { flowNum: 'N', test: 'Тест', extractDepth: 'Глубина извлечения', - extractAutomaticNumbering: 'Извлекать автоматическую нумерацию', - extractAutomaticNumberingTip: - 'Извлекать номера автоматически пронумерованных заголовков.', format: 'Формат', basic: 'базовый', advanced: 'продвинутый', @@ -2270,12 +2268,12 @@ export default { tokenizerRequired: 'Сначала добавьте узел индексатора', tokenizerDescription: 'Преобразует текст в нужную структуру данных (например, векторные эмбеддинги для embedding-поиска) в зависимости от выбранного метода поиска.', - splitter: 'Токен', - splitterDescription: - 'Разбивает текст на чанки по длине токенов с опциональными разделителями и перекрытием.', - hierarchicalMergerDescription: - 'Разбивает документы на секции по иерархии заголовков с regex для точного контроля.', - hierarchicalMerger: 'Заголовок', + tokenChunker: 'Токен-чанкер', + tokenChunkerDescription: + 'Разбивает текст на чанки по количеству токенов с настраиваемыми разделителями и перекрытием.', + titleChunker: 'Чанкер по заголовкам', + titleChunkerDescription: + 'Разбивает документы на секции по иерархии заголовков. Уровни задаются регулярными выражениями, а режим определяет структуру чанков.', extractor: 'Трансформер', extractorDescription: 'Использует LLM для извлечения структурированных данных из чанков — резюме, классификаций и т.д.', @@ -2286,16 +2284,48 @@ export default { spreadsheet: 'Электронная таблица', image: 'Изображение', email: 'Электронная почта', - 'text&markdown': 'Текст и разметка', - word: 'Word файл', + markdown: 'Markdown', + 'text&code': 'Текст и код', + html: 'HTML', + doc: 'DOC', + docx: 'DOCX', slides: 'PPTX', audio: 'Аудио', video: 'Видео', }, - fields: 'Поле', + fields: 'Поля', addParser: 'Добавить парсер', + rule: 'Правило', + addRule: 'Добавить правило', + group: 'Группировка', hierarchy: 'Иерархия', + addRegularExpressions: 'Добавить регулярное выражение', regularExpressions: 'Регулярные выражения', + hierarchyTip: `Строит дерево заголовков и создаёт самодостаточные чанки с полным путём по иерархии, например: Часть 1 › Глава 3 › Раздел 2 + содержимое. + +Подходит для строго структурированных текстов: законов, нормативных актов, договоров и технической документации, где важно положение каждого чанка в иерархии.`, + groupTip: `Разбивает документ на плоские группы по выбранному уровню заголовков и объединяет соседние небольшие секции. Родительские заголовки в чанки не добавляются. + +Подходит для книг, руководств, отчётов и статей, где важна смысловая связность соседних фрагментов.`, + includeHeadingContent: 'Отделять содержимое родительских заголовков', + includeHeadingContentTip: + 'Чанки будут содержать только путь заголовка и своё содержимое. Текст сразу после родительского заголовка будет сохранён в отдельном чанке.', + rootAsHeading: 'Использовать первый чанк как глобальный контекст', + rootAsHeadingTip: + 'Первый фрагмент будет использоваться как глобальный заголовок для всей иерархии. Это удобно для резюме и документов, где первая секция определяет предмет документа.', + enableMultiColumn: 'Распознавать многоколоночную разметку', + enableMultiColumnTip: + 'Определять колонки и сохранять правильный порядок чтения. Включите для документов с двумя колонками или газетной вёрсткой.', + removeToc: 'Удалять исходное оглавление', + removeTocTip: + 'Удалять оглавление исходного документа, чтобы оно не обрабатывалось как обычный текст и не попадало в чанки.', + removeHeaderFooter: 'Удалять колонтитулы', + flattenMediaToText: 'Отключить визуальную модель', + flattenMediaToTextTip: + 'Обрабатывать изображения и таблицы как обычный текст без улучшения визуальной моделью.', + extractAutomaticNumbering: 'Извлекать автоматическую нумерацию', + extractAutomaticNumberingTip: + 'Извлекать номера автоматически пронумерованных заголовков.', overlappedPercent: 'Процент перекрытия (%)', searchMethod: 'Метод поиска', searchMethodTip: `Определяет, как ищется контент — полнотекстом, эмбеддингами или обоими способами. diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsx index 5937ba3cf6d..cd41388ba12 100644 --- a/web/src/pages/agent/form/parser-form/index.tsx +++ b/web/src/pages/agent/form/parser-form/index.tsx @@ -148,7 +148,7 @@ function ParserItem({ >
- Parser {index + 1} + {t('flow.parser')} {index + 1} {index > 0 && (