Janady13/missionmatch / cmd/opportunity-extract/pdfworker.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/opportunity-extract/pdfworker.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/opportunity-extract/pdfworker.go
Original SHA-256	f26ab1a6143c0d8e910c04e159b8c227335eac2ee53f5c75df3e7d466e01f70e
Git blob SHA	4d3988c4a74d1fce67e9c4745b45cce0e2847966
Reading-copy SHA-256	f26ab1a6143c0d8e910c04e159b8c227335eac2ee53f5c75df3e7d466e01f70e

Original source ↗

Recovered source transcript

Download complete source text ↓111 lines · 4.5 KB

package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"time"
)

// THE PDF LIBRARY RUNS IN A CHILD PROCESS, and this file is the whole reason why.
//
// 07_TE6_Deliverables.pdf from Package B is 97,642 bytes and six pages. The extractor was OOM KILLED on it,
// exit 137, without printing a verdict for the document and without reaching its own DONE line. One small file
// took the entire batch down, which directly contradicts the promise at the top of main.go: a document this
// job cannot read is marked failed WITH A REASON while the run continues.
//
// MEASURED, NOT GUESSED. A diagnostic walked the document page by page: NewReader returns cleanly and reports
// six pages, and the process dies inside the FIRST page's GetPlainText before any text comes back. So the
// runaway allocation happens inside ONE library call.
//
// THAT RULES OUT EVERY IN PROCESS FIX. A page count bound does not help when page one is the problem. An output
// size bound does not help when the allocation is inside the callee. The existing panic recovery does not help
// either, because a process killed by the kernel's OOM killer never gets to run a deferred function, and Go
// cannot recover from memory exhaustion by design.
//
// SO THE CALL IS ISOLATED. The extractor re executes ITSELF in a worker mode that reads PDF bytes on stdin and
// writes text on stdout, under a wall clock deadline and an output cap. If the child dies, for any reason
// including being OOM killed, the parent simply learns that the library could not read this document and falls
// through to the hand rolled fallback reader, which does not exhaust memory. A dead child is a fact about one
// file, exactly as a panic already was.
//
// It re executes ITSELF rather than shelling out to a tool, so there is no new dependency and nothing to
// install, and the no Python rule is untouched.

// pdfWorkerFlag is the hidden mode. It is not documented in the usage line because no operator should run it.
const pdfWorkerFlag = "-pdf-text-worker"

// workerLimits are deliberately generous against a 32 MB document cap: an honest solicitation finishes well
// inside them, and the pathological case is stopped in seconds rather than taking the host down.
const (
	workerDeadline  = 90 * time.Second
	workerOutputCap = 64 << 20
)

// runPDFTextWorker is the child. It reads the whole document from stdin, extracts, and writes text to stdout.
// It returns an exit code rather than calling os.Exit so main stays testable.
func runPDFTextWorker() int {
	raw, err := io.ReadAll(io.LimitReader(os.Stdin, maxExtractBytes+1))
	if err != nil {
		fmt.Fprintf(os.Stderr, "pdf worker: read stdin: %v\n", err)
		return 2
	}
	text, ok := pdfTextViaLibraryInProcess(raw)
	if !ok {
		return 3
	}
	if _, err := os.Stdout.WriteString(text); err != nil {
		return 4
	}
	return 0
}

// pdfTextViaLibrary is what the extraction path calls. It hands the bytes to a child and interprets the child's
// fate. Every failure mode collapses to the same honest answer: this library could not read this document.
func pdfTextViaLibrary(raw []byte) (string, bool) {
	self, err := os.Executable()
	if err != nil {
		// Without a path to re execute there is no isolation available, so the fallback reader is used rather
		// than risking the batch on an unprotected library call.
		log.Printf("    pdf reader: cannot locate own binary for isolation (%v), using the fallback reader", err)
		return "", false
	}

	ctx, cancel := context.WithTimeout(context.Background(), workerDeadline)
	defer cancel()

	cmd := exec.CommandContext(ctx, self, pdfWorkerFlag)
	cmd.Stdin = bytes.NewReader(raw)
	var out, errBuf bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &errBuf
	applyWorkerLimits(cmd)

	runErr := cmd.Run()
	if ctx.Err() == context.DeadlineExceeded {
		log.Printf("    pdf reader: the isolated reader exceeded %s, treating the document as unreadable by the "+
			"library and falling back", workerDeadline)
		return "", false
	}
	if runErr != nil {
		// THIS IS THE OOM PATH, and it is now a survivable one. A killed child reports a signal here instead of
		// taking the parent with it.
		detail := runErr.Error()
		if s := errBuf.String(); s != "" {
			detail += ": " + s
		}
		log.Printf("    pdf reader: the isolated reader failed (%s), falling back", detail)
		return "", false
	}
	if out.Len() > workerOutputCap {
		log.Printf("    pdf reader: the isolated reader produced %d bytes, past the %d cap, falling back",
			out.Len(), workerOutputCap)
		return "", false
	}
	return out.String(), true
}