Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / cmd/opportunity-extract/oom_regression_test.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | cmd/opportunity-extract/oom_regression_test.go |
| Original SHA-256 | 7db952f7291cab6a84e1560e013cdfaf02bdb49dc0ac0b59c6f97d8576e63baf |
| Git blob SHA | 23c4213d18f4f4f329d2a424555a0a2e17204263 |
| Reading-copy SHA-256 | 7db952f7291cab6a84e1560e013cdfaf02bdb49dc0ac0b59c6f97d8576e63baf |
Recovered source transcript
Download complete source text ↓71 lines · 2.5 KB
package main
import (
"os"
"runtime"
"testing"
"time"
"github.com/missionworkforce/missionmatch/internal/modules/opportunities"
)
// A REGRESSION TEST FOR A DOCUMENT THAT KILLED THE EXTRACTOR.
//
// 07_TE6_Deliverables.pdf from Package B is 97,642 bytes and the extractor was OOM KILLED on it, exit 137. It
// never printed a verdict for the document and never reached its own DONE line, so a single 97 KB file took the
// whole batch down.
//
// THAT VIOLATES THE EXTRACTOR'S OWN STATED GUARANTEE, written at the top of main.go: "IT SURVIVES BAD INPUT
// RATHER THAN CRASHING ON IT ... A batch job that dies on the first bad file leaves every later file
// unprocessed and tells you about one problem instead of all of them." A document that cannot be read is
// supposed to be marked failed WITH A REASON while the run continues.
//
// The test is skipped unless the fixture is present, because a 97 KB binary does not belong in the repository.
// Point MM_TEST_TE6_PDF at a copy to run it.
func TestTE6DoesNotExhaustMemory(t *testing.T) {
path := os.Getenv("MM_TEST_TE6_PDF")
if path == "" {
t.Skip("set MM_TEST_TE6_PDF to the 07_TE6_Deliverables.pdf fixture to run this")
}
raw, err := os.ReadFile(path)
if err != nil {
t.Skipf("fixture unreadable: %v", err)
}
t.Logf("fixture is %d bytes", len(raw))
// A HARD CEILING, because the failure mode is the kernel killing the process rather than an error being
// returned. Without a ceiling this test reproduces the OOM instead of reporting it.
const ceilingMB = 512
done := make(chan struct{})
go func() {
defer close(done)
var m runtime.MemStats
for {
select {
case <-done:
return
default:
}
runtime.ReadMemStats(&m)
if m.HeapAlloc/(1<<20) > ceilingMB {
t.Errorf("heap passed %d MB while reading a %d byte PDF, which is the OOM this test guards",
ceilingMB, len(raw))
return
}
time.Sleep(20 * time.Millisecond)
}
}()
text, reason := textFrom(raw, "application/pdf", "07_TE6_Deliverables.pdf")
// EITHER OUTCOME IS ACCEPTABLE. A named refusal is the designed behaviour for a document this build cannot
// read; text is the better outcome. Being killed is the only unacceptable one.
if reason != "" {
t.Logf("refused with a named reason: %s. That is the designed graceful path", reason)
return
}
chunks := opportunities.ExtractDeterministic(text)
tables := opportunities.ExtractTables(text)
t.Logf("read %d bytes of text into %d chunks and %d tables", len(text), len(chunks), len(tables))
}