Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / internal/modules/opportunities/extract_tables_test.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | internal/modules/opportunities/extract_tables_test.go |
| Original SHA-256 | a8ae7393d1d875c740881fcd5ba91c1ff9007b5d7dcbb485b09b1fc676b328e3 |
| Git blob SHA | 70ba017b2c917e224fb6b79c23ff6a33be799d83 |
| Reading-copy SHA-256 | a8ae7393d1d875c740881fcd5ba91c1ff9007b5d7dcbb485b09b1fc676b328e3 |
Recovered source transcript
Download complete source text ↓181 lines · 7.6 KB
package opportunities
import (
"strings"
"testing"
)
// SECTION 5.1 LISTS TABLES among the facts deterministic extraction must find, alongside headings and
// numbered clauses. Until now it found neither tables nor the rates inside them, which is why the SCA wage
// determination in Package A dropped 231 of 232 spans: the prose gate correctly refused rows of codes and
// numbers, and nothing else was looking at them. A wage determination is the most staffing relevant document
// in a solicitation package, because it is the floor under every bill rate in the proposal.
//
// The fix is NOT a looser prose threshold. It is a SECOND, STRUCTURAL reader: a table row is recognised by
// its SHAPE, columns separated by a gutter, and validated by that shape rather than by looking like language.
// That keeps the prose gate exactly as strict as it was against decompressed font data, which is the failure
// it was written for.
// A real SCA wage determination, the shape the government actually publishes.
const wageDetermination = `WD 15-4191 (Rev.-32) was first posted on www.sam.gov on 12/24/2025
OCCUPATION CODE - TITLE FOOTNOTE RATE
01000 - Administrative Support And Clerical Occupations
01011 - Accounting Clerk I 17.23
01012 - Accounting Clerk II 19.34
01013 - Accounting Clerk III 21.62
01020 - Administrative Assistant 28.45
05000 - Automotive Service Occupations
05005 - Automobile Body Repairer, Fiberglass 27.19
05010 - Automotive Electrician 24.88
ALL OCCUPATIONS LISTED ABOVE RECEIVE THE FOLLOWING BENEFITS:
HEALTH & WELFARE: $5.36 per hour or $214.40 per week
`
// Decompressed font data, the thing the prose gate exists to refuse. It must not become a table either.
const fontNoise = `d0 d1 d2 endobj xref 0000000000 65535 f 0000000015 00000 n
/FontFile2 12 0 R /StemV 80 /XHeight 250 /FontBBox
BXbxRvXQhAdKZmPqTsWuYvIwJxKyLzMaNbOcPdQeRfSgThUiVjWk`
func TestExtractTablesFindsAWageDeterminationRateTable(t *testing.T) {
got := ExtractTables(wageDetermination)
if len(got) == 0 {
t.Fatal("no tables found in a wage determination, which is the document this reader exists for")
}
var rows int
for _, c := range got {
if c.Method != "table" {
t.Errorf("chunk method = %q, want table", c.Method)
}
if c.ContentHash == "" {
t.Error("a table chunk carries no content hash, so it cannot be cited")
}
if c.Confidence != 1.0 {
t.Errorf("confidence = %v, want 1.0: a literal copy is certain about what it copied", c.Confidence)
}
rows += strings.Count(c.Text, "\n") + 1
}
// The rate table has seven code rows plus its header. Requiring most of them keeps this honest without
// pinning the assertion to one exact split.
if rows < 6 {
t.Errorf("table rows recovered = %d, want at least 6 from a table with seven rate lines", rows)
}
}
func TestExtractTablesRefusesFontNoise(t *testing.T) {
if got := ExtractTables(fontNoise); len(got) != 0 {
t.Fatalf("font noise produced %d table(s). The structural reader must not become the hole in the "+
"prose gate, or the whole point of refusing garbage chunks is lost:\n%q", len(got), got[0].Text)
}
}
func TestExtractTablesRefusesOrdinaryProse(t *testing.T) {
prose := `The contractor shall provide continuous monitoring of the enclave in accordance with the
customer's internal baseline. All personnel shall hold a TS/SCI clearance with a CI polygraph, and
travel is estimated at 15 percent of the period of performance.`
if got := ExtractTables(prose); len(got) != 0 {
t.Fatalf("prose was read as a table (%d), which would duplicate every paragraph as tabular data", len(got))
}
}
func TestExtractTablesNeedsMoreThanOneRow(t *testing.T) {
// One spaced line is a heading or a wrapped sentence, not a table. Treating it as one would turn every
// column aligned title in a solicitation into a fake table.
single := "OCCUPATION CODE - TITLE FOOTNOTE RATE\n\nSome following prose here."
if got := ExtractTables(single); len(got) != 0 {
t.Fatalf("a single gutter separated line became a table (%d)", len(got))
}
}
func TestExtractRateRowsParsesCodeTitleAndRate(t *testing.T) {
rows := ExtractRateRows(wageDetermination)
if len(rows) < 6 {
t.Fatalf("rate rows = %d, want at least 6", len(rows))
}
byCode := map[string]RateRow{}
for _, r := range rows {
byCode[r.Code] = r
}
got, ok := byCode["01011"]
if !ok {
t.Fatal("occupation code 01011 was not extracted")
}
if got.Title != "Accounting Clerk I" {
t.Errorf("title = %q, want %q", got.Title, "Accounting Clerk I")
}
if got.HourlyRate != "17.23" {
t.Errorf("rate = %q, want 17.23", got.HourlyRate)
}
// A category header line carries no rate and must not be invented as zero, because a zero rate in a
// staffing tool reads as free labour rather than as missing data.
if _, present := byCode["01000"]; present {
t.Error("a category header with no rate was emitted as a rate row")
}
}
func TestExtractRateRowsIsDeterministic(t *testing.T) {
a := ExtractRateRows(wageDetermination)
b := ExtractRateRows(wageDetermination)
if len(a) != len(b) {
t.Fatalf("same input produced %d then %d rows", len(a), len(b))
}
for i := range a {
if a[i] != b[i] {
t.Fatalf("row %d differs between runs: %+v vs %+v", i, a[i], b[i])
}
}
}
func TestExtractTablesIsDeterministic(t *testing.T) {
a := ExtractTables(wageDetermination)
b := ExtractTables(wageDetermination)
if len(a) != len(b) {
t.Fatalf("same input produced %d then %d tables", len(a), len(b))
}
for i := range a {
if a[i].ContentHash != b[i].ContentHash {
t.Fatalf("table %d hash differs between runs", i)
}
}
}
// THE MOJIBAKE CASE, added after running the reader against the REAL SCA wage determination. That PDF's text
// layer decodes to character soup because the font carries no usable map, and the soup contains tabs, so it
// satisfied every structural test and produced "tables" of content like "(0 CC" and "CC A3".
//
// A STRUCTURAL READER THAT BYPASSES THE PROSE GATE MUST CARRY ITS OWN QUALITY BAR, or it simply becomes the
// hole in the gate. Garbage chunks are worse than no chunks because a reviewer cites them.
func TestExtractTablesRefusesMojibakeThatHappensToHaveGutters(t *testing.T) {
// Taken from the actual chunks the first version stored from the real document.
mojibake := "(0\tCC\t(\tE\n" +
"CC\tA3\t+8,F\n" +
"\t(008:(3*9\tCC\n" +
"%* +\"% !\"!,'()-./)0)\t%\"1\t++232\n"
if got := ExtractTables(mojibake); len(got) != 0 {
t.Fatalf("mojibake produced %d table(s). This is the regression that admitted noise the prose gate "+
"had correctly refused:\n%q", len(got), got[0].Text)
}
}
func TestExtractTablesStillAcceptsARealRateTableAfterTheReadabilityBar(t *testing.T) {
// The bar must not cost the legitimate case, so the wage determination fixture is asserted again here.
if got := ExtractTables(wageDetermination); len(got) == 0 {
t.Fatal("the readability bar refused a real rate table, which is the document this reader exists for")
}
}
func TestBlockIsReadableNeedsAWordWithAVowel(t *testing.T) {
if blockIsReadable([]string{"CC\tA3", "BX\tQZ"}) {
t.Error("consonant runs with no vowel were treated as readable")
}
if !blockIsReadable([]string{"01011 - Accounting Clerk I\t17.23"}) {
t.Error("a real labor category row was treated as unreadable")
}
// A numeric only table with ONE legible header is still evidence.
if !blockIsReadable([]string{"RATE\tFOOTNOTE", "17.23\t2"}) {
t.Error("a numeric table with a legible header was refused")
}
}