Janady13/missionmatch / internal/platform/docextract/docextract.go

1// Package docextract turns a downloaded federal attachment into plain text for the deterministic 2// mission and labor classifier. It is pure Go with no cgo and no network, so it builds under 3// CGO_ENABLED=0 and GOFIPS140=v1.0.0 and never reaches a Python tool. It dispatches on the file 4// extension: PDF through a pure Go reader, DOCX through the standard library zip and xml packages, 5// HTML through the x/net tokenizer, and plain text (or any unknown extension) as the raw bytes. 6// 7// This package is imported ONLY by the federalsignal sync path, never by cmd/api. The PDF reader drags 8// in stdlib crypto for encrypted PDF support (a dead path here, the attachments are public), so keeping 9// it out of the serving binary keeps the API SBOM clean. See docs/STACK_DECISIONS.md. 10package docextract 11 12import ( 13 "bytes" 14 "context" 15 "errors" 16 "fmt" 17 "io" 18 "strings" 19 20 "archive/zip" 21 "encoding/xml" 22 23 "golang.org/x/net/html" 24 25 "github.com/ledongthuc/pdf" 26) 27 28// maxTextBytes caps the extracted text so one oversized attachment can never blow up memory or the 29// classifier. One megabyte of text is far more than any labor mix or mission keyword needs. 30const maxTextBytes = 1 << 20 31 32// ErrUnsupported names the unsupported file type condition. ExtractText itself is permissive: an 33// unknown extension falls back to treating the bytes as plain text rather than failing, because a 34// federal attachment with an odd extension is more useful mined than dropped. ErrUnsupported is 35// exported so a stricter caller (or a future strict mode) can construct and branch on the typed error 36// without redefining it. 37var ErrUnsupported = errors.New("docextract: unsupported file type") 38 39// ErrParse wraps any failure to parse a structurally broken file (a corrupt PDF, a non zip DOCX, and so 40// on). Detect with errors.Is(err, ErrParse). The underlying cause is wrapped for diagnostics, the file 41// bytes themselves are never echoed so no attachment content leaks into a log. 42var ErrParse = errors.New("docextract: parse failed") 43 44// ExtractText returns the plain text of data, dispatched on the lowercased extension of filename. It 45// caps the result at maxTextBytes. On a corrupt or unsupported file it returns a typed error wrapping 46// ErrParse or ErrUnsupported. 47func ExtractText(ctx context.Context, filename string, data []byte) (string, error) { 48 ext := lowerExt(filename) 49 switch ext { 50 case ".pdf": 51 return capText(extractPDF(ctx, data)) 52 case ".docx": 53 return capText(extractDOCX(data)) 54 case ".htm", ".html": 55 return capText(extractHTML(data)) 56 case ".txt", "": 57 return capText(string(data), nil) 58 default: 59 // Unknown extensions fall back to treating the bytes as text rather than failing, since a federal 60 // attachment with an odd extension (for example a .rtf saved as plain text) is more useful mined 61 // than dropped. A caller that needs strictness can compare the extension before calling. 62 return capText(string(data), nil) 63 } 64} 65 66// lowerExt returns the lowercased extension of name including the leading dot, or empty if none. 67func lowerExt(name string) string { 68 i := strings.LastIndexByte(name, '.') 69 if i < 0 || i == len(name)-1 { 70 return "" 71 } 72 // Reject a dot that is part of a path segment with no real extension (for example a trailing slash). 73 if strings.ContainsAny(name[i+1:], "/\\") { 74 return "" 75 } 76 return strings.ToLower(name[i:]) 77} 78 79// cap truncates s to maxTextBytes, on a UTF-8 rune boundary, and passes any error through unchanged. 80func capText(s string, err error) (string, error) { 81 if err != nil { 82 return "", err 83 } 84 if len(s) <= maxTextBytes { 85 return s, nil 86 } 87 cut := maxTextBytes 88 // Back up to a rune boundary so the cap never splits a multibyte rune. 89 for cut > 0 && !utf8Start(s[cut]) { 90 cut-- 91 } 92 return s[:cut], nil 93} 94 95// utf8Start reports whether b is the first byte of a UTF-8 sequence (not a continuation byte). 96func utf8Start(b byte) bool { return b&0xC0 != 0x80 } 97 98// extractPDF reads every page's plain text from a PDF held entirely in memory. 99func extractPDF(ctx context.Context, data []byte) (string, error) { 100 if len(data) == 0 { 101 return "", fmt.Errorf("%w: empty pdf", ErrParse) 102 } 103 r, err := pdf.NewReader(bytes.NewReader(data), int64(len(data))) 104 if err != nil { 105 // Wrap with ErrParse only, the cause may carry structural detail but never the file bytes. 106 return "", fmt.Errorf("%w: pdf reader: %v", ErrParse, err) 107 } 108 rd, err := r.GetPlainText() 109 if err != nil { 110 return "", fmt.Errorf("%w: pdf text: %v", ErrParse, err) 111 } 112 // Bound the copy so a hostile or malformed PDF that claims a huge text length cannot exhaust memory. 113 var sb strings.Builder 114 if _, err := io.Copy(&sb, io.LimitReader(rd, maxTextBytes+1)); err != nil { 115 return "", fmt.Errorf("%w: pdf read: %v", ErrParse, err) 116 } 117 _ = ctx 118 return sb.String(), nil 119} 120 121// extractDOCX reads word/document.xml from the DOCX zip and joins the text runs with spaces. 122func extractDOCX(data []byte) (string, error) { 123 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 124 if err != nil { 125 return "", fmt.Errorf("%w: docx zip: %v", ErrParse, err) 126 } 127 var doc *zip.File 128 for _, f := range zr.File { 129 if f.Name == "word/document.xml" { 130 doc = f 131 break 132 } 133 } 134 if doc == nil { 135 return "", fmt.Errorf("%w: docx missing word/document.xml", ErrParse) 136 } 137 rc, err := doc.Open() 138 if err != nil { 139 return "", fmt.Errorf("%w: docx open part: %v", ErrParse, err) 140 } 141 defer func() { _ = rc.Close() }() 142 143 dec := xml.NewDecoder(io.LimitReader(rc, 64<<20)) // bound the part so a zip bomb cannot run away 144 var sb strings.Builder 145 inText := false 146 for { 147 tok, err := dec.Token() 148 if err == io.EOF { 149 break 150 } 151 if err != nil { 152 return "", fmt.Errorf("%w: docx xml: %v", ErrParse, err) 153 } 154 switch t := tok.(type) { 155 case xml.StartElement: 156 // A WordprocessingML text run is the <w:t> element. Match on the local name so the namespace 157 // prefix (w) does not have to be hard coded. 158 if t.Name.Local == "t" { 159 inText = true 160 } 161 case xml.EndElement: 162 if t.Name.Local == "t" { 163 inText = false 164 sb.WriteByte(' ') // join adjacent runs with a space so words do not collide 165 } 166 case xml.CharData: 167 if inText { 168 sb.Write(t) 169 } 170 } 171 if sb.Len() > maxTextBytes { 172 break 173 } 174 } 175 return strings.TrimSpace(sb.String()), nil 176} 177 178// extractHTML returns the visible text of an HTML document, dropping the content of script and style 179// elements and collapsing tags to whitespace. 180func extractHTML(data []byte) (string, error) { 181 z := html.NewTokenizer(bytes.NewReader(data)) 182 var sb strings.Builder 183 skipDepth := 0 // > 0 while inside a script or style subtree 184 for { 185 tt := z.Next() 186 if tt == html.ErrorToken { 187 if err := z.Err(); err != nil && err != io.EOF { 188 return "", fmt.Errorf("%w: html: %v", ErrParse, err) 189 } 190 break 191 } 192 switch tt { 193 case html.StartTagToken: 194 name, _ := z.TagName() 195 if isSkippable(name) { 196 skipDepth++ 197 } 198 case html.EndTagToken: 199 name, _ := z.TagName() 200 if isSkippable(name) && skipDepth > 0 { 201 skipDepth-- 202 } 203 case html.TextToken: 204 if skipDepth == 0 { 205 txt := strings.TrimSpace(string(z.Text())) 206 if txt != "" { 207 if sb.Len() > 0 { 208 sb.WriteByte(' ') 209 } 210 sb.WriteString(txt) 211 } 212 } 213 } 214 if sb.Len() > maxTextBytes { 215 break 216 } 217 } 218 return sb.String(), nil 219} 220 221// isSkippable reports whether an element's text content must be dropped (script and style carry code, 222// never visible prose). 223func isSkippable(name []byte) bool { 224 switch string(bytes.ToLower(name)) { 225 case "script", "style": 226 return true 227 default: 228 return false 229 } 230} 231