Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / internal/modules/opportunities/extract_tables.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 720d47fae476b0d38477ae4401697d372b1ea2bc |
| Branch | feature/opportunity-to-outcome |
| Path or archive member | internal/modules/opportunities/extract_tables.go |
| Original SHA-256 | 1ba36a5694e2e510022e6e53601d0dd11d3c7d236cf26a6fce0d2102cca07899 |
| Git blob SHA | 725523e3caa4bfa08541010fd526550b1ef6a84f |
| Reading-copy SHA-256 | 1ba36a5694e2e510022e6e53601d0dd11d3c7d236cf26a6fce0d2102cca07899 |
Recovered source transcript
Download complete source text ↓254 lines · 9.2 KB
package opportunities
import (
"regexp"
"strings"
"unicode"
)
// TABLE EXTRACTION, directive Section 5.1, which lists tables among the facts deterministic extraction must
// find. This is a SECOND reader alongside ExtractDeterministic, not a relaxation of the prose gate.
//
// WHY IT IS SEPARATE AND STRUCTURAL. The prose gate refuses spans that do not look like language, and it is
// right to: decompressed font programs and image streams otherwise arrive as chunks that a reviewer would
// cite. A rate table legitimately looks nothing like language, so the answer is not a lower threshold, which
// would readmit the noise. The answer is to recognise a table by its SHAPE and validate it by that shape.
//
// THE SHAPE. A table row is a line whose cells are separated by a GUTTER, two or more spaces or a tab. That
// single property does most of the work, because prose is single spaced and font blobs carry no gutters at
// all. Everything else here exists to stop the two ways this could still go wrong: one aligned heading being
// promoted to a table, and a run of garbage that happens to contain wide whitespace.
//
// Everything in this file is pure. Same input, same output, forever.
// RateRow is one labor category and its floor rate, lifted from a wage determination. Section 5.1 asks for
// labor categories and dollar values, and on a wage determination they live in the same row.
//
// HourlyRate is kept as a STRING deliberately. It is a literal copy of what the document says, and the
// moment it becomes a float it stops being a citation and starts being a calculation. Scoring math uses
// shopspring/decimal elsewhere for the same reason.
type RateRow struct {
Code string // occupation code, for example 01011
Title string // labor category as written, for example Accounting Clerk I
HourlyRate string // the literal rate text, for example 17.23
}
var (
// A gutter: two or more spaces, or any tab. One space is prose.
gutterRe = regexp.MustCompile(`\t+| {2,}`)
// A wage determination rate line: an occupation code, a title, then a rate across a gutter. The rate is
// required, which is what keeps a CATEGORY HEADER such as "01000 - Administrative Support" out of the
// results. Emitting that as a row with a zero rate would read as free labour rather than as missing data.
rateRowRe = regexp.MustCompile(`(?m)^\s*(\d{4,6})\s*-\s*(\S.*?)(?:\t+| {2,})(?:.*?(?:\t+| {2,}))?\$?(\d{1,3}\.\d{2})\s*$`)
)
// ExtractTables finds tabular regions and returns each as one citable chunk with its rows intact. Rows are
// kept together rather than split one chunk per row, because a rate divorced from its column header is not
// evidence of anything.
func ExtractTables(text string) []Chunk {
if strings.TrimSpace(text) == "" {
return nil
}
page := 1
lines := strings.Split(text, "\n")
var out []Chunk
var block []string
var realRows int
softBreaks := 0
flush := func() {
// TWO REAL ROWS MINIMUM. One gutter separated line is a heading or a wrapped sentence, and promoting
// it would turn every column aligned title in a solicitation into a fake table.
//
// AND THE BLOCK MUST BE READABLE. This second bar was added after the reader was run against the REAL
// SCA wage determination, whose text layer decodes to mojibake because the PDF uses an embedded font
// with no usable character map. Structurally that mojibake contains gutters, so it produced "tables"
// of content like "(0 CC" and "CC A3": garbage that had passed the shape test.
//
// THAT WAS A REGRESSION IN THE GUARANTEE THE PROSE GATE EXISTS TO HOLD, that a garbage chunk is worse
// than no chunk because a reviewer cites it. A structural reader that bypasses the prose gate must
// carry its own quality bar or it becomes the hole in it.
if realRows >= 2 && blockIsReadable(block) {
body := strings.TrimRight(strings.Join(block, "\n"), "\n")
if strings.TrimSpace(body) != "" {
out = append(out, newChunk(tableLabel(block), page, body, "table"))
}
}
block = nil
realRows = 0
softBreaks = 0
}
for _, raw := range lines {
line := strings.TrimRight(raw, " \t\r")
if strings.TrimSpace(line) == "" {
// A blank line ends a table. This is what stops the benefits prose after a wage determination
// from being absorbed into the rate table above it.
flush()
continue
}
if isTableRow(line) {
block = append(block, line)
realRows++
softBreaks = 0
continue
}
// A NON ROW LINE INSIDE A TABLE. Real tables carry sub headers that span the full width and therefore
// have no gutter, for example "01000 - Administrative Support And Clerical Occupations". Dropping the
// block at the first one would shatter a single rate table into fragments. ONE such line is absorbed;
// a second consecutive one means the table is over and prose has resumed.
if realRows > 0 && softBreaks == 0 && len(line) <= 120 {
block = append(block, line)
softBreaks++
continue
}
flush()
}
flush()
return out
}
// tableLabel names a table by the first line that looks like a header, so a citation says which table it came
// from rather than only that it came from a table.
func tableLabel(block []string) string {
for _, l := range block {
cells := splitCells(l)
if len(cells) < 2 {
continue
}
letters := 0
for _, r := range l {
if unicode.IsLetter(r) {
letters++
}
}
// A header row is mostly words. A data row carries numbers.
if letters*2 > len(l) {
return strings.Join(cells, " | ")
}
}
return "table"
}
// blockIsReadable asks whether a structurally table shaped block contains anything a human could actually
// read. A rate table names labor categories, and a labor category is a WORD: several letters, containing a
// vowel. Decompressed font data and mis mapped character codes produce short runs of consonants and digits
// that satisfy every shape test and mean nothing.
//
// ONE READABLE CELL IS ENOUGH. A table of pure numbers with one legible header is still evidence; a table with
// no legible cell anywhere is not a table, it is noise with tabs in it.
func blockIsReadable(block []string) bool {
const minWordLetters = 4
for _, line := range block {
for _, cell := range splitCells(line) {
letters, vowels, run := 0, 0, 0
best := 0
for _, r := range strings.ToLower(cell) {
if r >= 'a' && r <= 'z' {
letters++
run++
if run > best {
best = run
}
if strings.ContainsRune("aeiouy", r) {
vowels++
}
continue
}
run = 0
}
// A word: a long enough unbroken letter run, and at least one vowel in the cell. "Accounting"
// passes. "CC", "A3" and "BXbxRvXQ" do not.
if best >= minWordLetters && vowels > 0 && letters >= minWordLetters {
return true
}
}
}
return false
}
// isTableRow decides whether one line is a row of a table. It is deliberately structural: it asks how the
// line is SHAPED, never whether it reads like language.
func isTableRow(line string) bool {
cells := splitCells(line)
if len(cells) < 2 {
return false
}
// THE DENOMINATOR IS CONTENT, NOT THE WHOLE LINE. Measuring against every character counts the gutter
// itself as evidence against the row, and a table's gutters are wide by definition. The first version did
// exactly that and rejected the wage determination's own column header, which is 42 spaces wide in the
// middle: 33 alphanumerics in 83 characters scored 0.40 and failed a 0.45 floor. What matters is the
// composition of the CONTENT characters, so spaces are excluded from the denominator.
var alnum, symbols, content, longestRun, run int
for _, r := range line {
switch {
case unicode.IsLetter(r), unicode.IsDigit(r):
alnum++
content++
run++
if run > longestRun {
longestRun = run
}
case unicode.IsSpace(r):
run = 0
case strings.ContainsRune(".,;:()/-'\"%$&+#*[]", r):
content++
run = 0
default:
symbols++
content++
run = 0
}
}
total := content
if total == 0 {
return false
}
// A row is mostly words and numbers. Font data is mostly neither.
if float64(alnum)/float64(total) < 0.45 {
return false
}
// Symbols are rare in a table and constant in a decompressed stream.
if float64(symbols)/float64(total) > 0.12 {
return false
}
// An unbroken forty character alphanumeric run is a base64 like blob, never a labor category.
if longestRun > 40 {
return false
}
return true
}
func splitCells(line string) []string {
parts := gutterRe.Split(strings.TrimSpace(line), -1)
out := make([]string, 0, len(parts))
for _, p := range parts {
if s := strings.TrimSpace(p); s != "" {
out = append(out, s)
}
}
return out
}
// ExtractRateRows lifts labor category and rate pairs out of a wage determination. This is the part of
// Section 5.1 that a staffing product actually prices against: the wage determination is the legal floor
// under every bill rate in the proposal.
func ExtractRateRows(text string) []RateRow {
if strings.TrimSpace(text) == "" {
return nil
}
ms := rateRowRe.FindAllStringSubmatch(text, -1)
out := make([]RateRow, 0, len(ms))
for _, m := range ms {
title := strings.TrimSpace(m[2])
// Trim a trailing footnote marker so the title is the category and nothing else.
title = strings.TrimRight(title, " .*")
if title == "" {
continue
}
out = append(out, RateRow{Code: m[1], Title: title, HourlyRate: m[3]})
}
return out
}