Janady13/missionmatch / internal/platform/docextract/docextract_test.go

1package docextract 2 3import ( 4 "archive/zip" 5 "bytes" 6 "context" 7 "errors" 8 "fmt" 9 "os" 10 "path/filepath" 11 "strings" 12 "testing" 13) 14 15// TestMain regenerates the test fixtures from code before the suite runs, so the fixtures are fully 16// reproducible (no opaque binary blobs that only a human could recreate). The generated files are also 17// committed under testdata so the suite still runs if regeneration were ever skipped. Each generator 18// builds a real file of its format: a valid single page PDF, a valid DOCX zip, a valid HTML document, 19// and a plain text file. 20func TestMain(m *testing.M) { 21 if err := writeFixtures(); err != nil { 22 fmt.Fprintf(os.Stderr, "docextract: write fixtures: %v\n", err) 23 os.Exit(1) 24 } 25 os.Exit(m.Run()) 26} 27 28func writeFixtures() error { 29 if err := os.MkdirAll("testdata", 0o755); err != nil { 30 return err 31 } 32 if err := os.WriteFile(filepath.Join("testdata", "sample.pdf"), buildMinimalPDF("SIGINT Analyst"), 0o644); err != nil { 33 return err 34 } 35 docx, err := buildMinimalDOCX("ISSO") 36 if err != nil { 37 return err 38 } 39 if err := os.WriteFile(filepath.Join("testdata", "sample.docx"), docx, 0o644); err != nil { 40 return err 41 } 42 html := "<!doctype html><html><head><title>Roles</title><style>p{color:red}</style></head>" + 43 "<body><h1>Open Positions</h1><p>COMSEC Custodian</p><script>var x=1;</script></body></html>" 44 if err := os.WriteFile(filepath.Join("testdata", "sample.html"), []byte(html), 0o644); err != nil { 45 return err 46 } 47 if err := os.WriteFile(filepath.Join("testdata", "sample.txt"), []byte("Cyber Engineer needed for the program."), 0o644); err != nil { 48 return err 49 } 50 return nil 51} 52 53// buildMinimalPDF builds a single page PDF whose content stream renders the given text. The cross 54// reference table offsets are computed from the running byte length so the file is structurally valid 55// and a pure Go reader can parse it. 56func buildMinimalPDF(text string) []byte { 57 var buf bytes.Buffer 58 var offsets []int 59 mark := func() { offsets = append(offsets, buf.Len()) } 60 61 buf.WriteString("%PDF-1.4\n") 62 mark() 63 buf.WriteString("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n") 64 mark() 65 buf.WriteString("2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n") 66 mark() 67 buf.WriteString("3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>\nendobj\n") 68 content := fmt.Sprintf("BT /F1 24 Tf 72 700 Td (%s) Tj ET", text) 69 mark() 70 buf.WriteString(fmt.Sprintf("4 0 obj\n<< /Length %d >>\nstream\n%s\nendstream\nendobj\n", len(content), content)) 71 mark() 72 buf.WriteString("5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n") 73 74 xrefStart := buf.Len() 75 buf.WriteString("xref\n0 6\n0000000000 65535 f \n") 76 for _, off := range offsets { 77 buf.WriteString(fmt.Sprintf("%010d 00000 n \n", off)) 78 } 79 buf.WriteString("trailer\n<< /Size 6 /Root 1 0 R >>\n") 80 buf.WriteString(fmt.Sprintf("startxref\n%d\n", xrefStart)) 81 buf.WriteString("%%EOF\n") 82 return buf.Bytes() 83} 84 85// buildMinimalDOCX builds a real minimal Office Open XML document (a zip with the three parts a reader 86// requires: the content types map, the package relationships, and the main document part) whose single 87// paragraph carries the given text. 88func buildMinimalDOCX(text string) ([]byte, error) { 89 var buf bytes.Buffer 90 zw := zip.NewWriter(&buf) 91 92 parts := []struct { 93 name, body string 94 }{ 95 { 96 "[Content_Types].xml", 97 `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` + 98 `<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">` + 99 `<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>` + 100 `<Default Extension="xml" ContentType="application/xml"/>` + 101 `<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>` + 102 `</Types>`, 103 }, 104 { 105 "_rels/.rels", 106 `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` + 107 `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">` + 108 `<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>` + 109 `</Relationships>`, 110 }, 111 { 112 "word/document.xml", 113 `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` + 114 `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">` + 115 `<w:body><w:p><w:r><w:t>` + text + ` Officer</w:t></w:r></w:p>` + 116 `<w:p><w:r><w:t>and a second paragraph</w:t></w:r></w:p></w:body>` + 117 `</w:document>`, 118 }, 119 } 120 for _, p := range parts { 121 w, err := zw.Create(p.name) 122 if err != nil { 123 return nil, err 124 } 125 if _, err := w.Write([]byte(p.body)); err != nil { 126 return nil, err 127 } 128 } 129 if err := zw.Close(); err != nil { 130 return nil, err 131 } 132 return buf.Bytes(), nil 133} 134 135func TestExtractText(t *testing.T) { 136 for _, tc := range []struct{ file, want string }{ 137 {"testdata/sample.pdf", "SIGINT Analyst"}, 138 {"testdata/sample.docx", "ISSO"}, 139 {"testdata/sample.html", "COMSEC Custodian"}, 140 {"testdata/sample.txt", "Cyber Engineer"}, 141 } { 142 b, err := os.ReadFile(tc.file) 143 if err != nil { 144 t.Fatalf("read %s: %v", tc.file, err) 145 } 146 got, err := ExtractText(context.Background(), tc.file, b) 147 if err != nil { 148 t.Fatalf("ExtractText(%s): %v", tc.file, err) 149 } 150 if !strings.Contains(got, tc.want) { 151 t.Fatalf("ExtractText(%s) missing %q in %q", tc.file, tc.want, got) 152 } 153 } 154} 155 156// TestExtractTextDOCXJoinsRuns proves the DOCX path joins separate text runs with a space so adjacent 157// paragraphs do not collide into one word. 158func TestExtractTextDOCXJoinsRuns(t *testing.T) { 159 b, err := os.ReadFile("testdata/sample.docx") 160 if err != nil { 161 t.Fatalf("read docx: %v", err) 162 } 163 got, err := ExtractText(context.Background(), "sample.docx", b) 164 if err != nil { 165 t.Fatalf("ExtractText docx: %v", err) 166 } 167 if !strings.Contains(got, "ISSO Officer") { 168 t.Fatalf("docx runs not joined as expected: %q", got) 169 } 170 if !strings.Contains(got, "second paragraph") { 171 t.Fatalf("docx second paragraph missing: %q", got) 172 } 173} 174 175// TestExtractTextHTMLDropsScriptAndStyle proves the HTML path returns visible text only, never the 176// content of script or style elements. 177func TestExtractTextHTMLDropsScriptAndStyle(t *testing.T) { 178 html := []byte(`<html><head><style>body{color:red}</style></head><body><p>Visible Role</p><script>secret()</script></body></html>`) 179 got, err := ExtractText(context.Background(), "x.html", html) 180 if err != nil { 181 t.Fatalf("ExtractText html: %v", err) 182 } 183 if !strings.Contains(got, "Visible Role") { 184 t.Fatalf("visible text missing: %q", got) 185 } 186 if strings.Contains(got, "secret()") || strings.Contains(got, "color:red") { 187 t.Fatalf("script or style content leaked into output: %q", got) 188 } 189} 190 191// TestExtractTextUnknownIsRaw proves an unknown extension returns the raw bytes as a string. 192func TestExtractTextUnknownIsRaw(t *testing.T) { 193 got, err := ExtractText(context.Background(), "notes.dat", []byte("raw payload")) 194 if err != nil { 195 t.Fatalf("ExtractText unknown: %v", err) 196 } 197 if got != "raw payload" { 198 t.Fatalf("unknown extension should return raw bytes, got %q", got) 199 } 200} 201 202// TestExtractTextCorruptPDF proves a corrupt PDF returns a wrapped parse error, not a panic. 203func TestExtractTextCorruptPDF(t *testing.T) { 204 _, err := ExtractText(context.Background(), "broken.pdf", []byte("%PDF-1.4 this is not a real pdf body")) 205 if err == nil { 206 t.Fatal("expected an error on a corrupt pdf") 207 } 208 if !errors.Is(err, ErrParse) { 209 t.Fatalf("corrupt pdf error should wrap ErrParse, got %v", err) 210 } 211} 212 213// TestExtractTextCorruptDOCX proves a non zip DOCX returns a wrapped parse error. 214func TestExtractTextCorruptDOCX(t *testing.T) { 215 _, err := ExtractText(context.Background(), "broken.docx", []byte("not a zip at all")) 216 if err == nil { 217 t.Fatal("expected an error on a corrupt docx") 218 } 219 if !errors.Is(err, ErrParse) { 220 t.Fatalf("corrupt docx error should wrap ErrParse, got %v", err) 221 } 222} 223 224// TestExtractTextCap proves the output is capped at maxTextBytes. 225func TestExtractTextCap(t *testing.T) { 226 big := bytes.Repeat([]byte("A"), maxTextBytes+5000) 227 got, err := ExtractText(context.Background(), "big.txt", big) 228 if err != nil { 229 t.Fatalf("ExtractText big txt: %v", err) 230 } 231 if len(got) > maxTextBytes { 232 t.Fatalf("output not capped: got %d bytes, cap %d", len(got), maxTextBytes) 233 } 234} 235