package main import ( "bytes" "context" "fmt" "io" "log" "os" "os/exec" "time" ) // THE PDF LIBRARY RUNS IN A CHILD PROCESS, and this file is the whole reason why. // // 07_TE6_Deliverables.pdf from Package B is 97,642 bytes and six pages. The extractor was OOM KILLED on it, // exit 137, without printing a verdict for the document and without reaching its own DONE line. One small file // took the entire batch down, which directly contradicts the promise at the top of main.go: a document this // job cannot read is marked failed WITH A REASON while the run continues. // // MEASURED, NOT GUESSED. A diagnostic walked the document page by page: NewReader returns cleanly and reports // six pages, and the process dies inside the FIRST page's GetPlainText before any text comes back. So the // runaway allocation happens inside ONE library call. // // THAT RULES OUT EVERY IN PROCESS FIX. A page count bound does not help when page one is the problem. An output // size bound does not help when the allocation is inside the callee. The existing panic recovery does not help // either, because a process killed by the kernel's OOM killer never gets to run a deferred function, and Go // cannot recover from memory exhaustion by design. // // SO THE CALL IS ISOLATED. The extractor re executes ITSELF in a worker mode that reads PDF bytes on stdin and // writes text on stdout, under a wall clock deadline and an output cap. If the child dies, for any reason // including being OOM killed, the parent simply learns that the library could not read this document and falls // through to the hand rolled fallback reader, which does not exhaust memory. A dead child is a fact about one // file, exactly as a panic already was. // // It re executes ITSELF rather than shelling out to a tool, so there is no new dependency and nothing to // install, and the no Python rule is untouched. // pdfWorkerFlag is the hidden mode. It is not documented in the usage line because no operator should run it. const pdfWorkerFlag = "-pdf-text-worker" // workerLimits are deliberately generous against a 32 MB document cap: an honest solicitation finishes well // inside them, and the pathological case is stopped in seconds rather than taking the host down. const ( workerDeadline = 90 * time.Second workerOutputCap = 64 << 20 ) // runPDFTextWorker is the child. It reads the whole document from stdin, extracts, and writes text to stdout. // It returns an exit code rather than calling os.Exit so main stays testable. func runPDFTextWorker() int { raw, err := io.ReadAll(io.LimitReader(os.Stdin, maxExtractBytes+1)) if err != nil { fmt.Fprintf(os.Stderr, "pdf worker: read stdin: %v\n", err) return 2 } text, ok := pdfTextViaLibraryInProcess(raw) if !ok { return 3 } if _, err := os.Stdout.WriteString(text); err != nil { return 4 } return 0 } // pdfTextViaLibrary is what the extraction path calls. It hands the bytes to a child and interprets the child's // fate. Every failure mode collapses to the same honest answer: this library could not read this document. func pdfTextViaLibrary(raw []byte) (string, bool) { self, err := os.Executable() if err != nil { // Without a path to re execute there is no isolation available, so the fallback reader is used rather // than risking the batch on an unprotected library call. log.Printf(" pdf reader: cannot locate own binary for isolation (%v), using the fallback reader", err) return "", false } ctx, cancel := context.WithTimeout(context.Background(), workerDeadline) defer cancel() cmd := exec.CommandContext(ctx, self, pdfWorkerFlag) cmd.Stdin = bytes.NewReader(raw) var out, errBuf bytes.Buffer cmd.Stdout = &out cmd.Stderr = &errBuf applyWorkerLimits(cmd) runErr := cmd.Run() if ctx.Err() == context.DeadlineExceeded { log.Printf(" pdf reader: the isolated reader exceeded %s, treating the document as unreadable by the "+ "library and falling back", workerDeadline) return "", false } if runErr != nil { // THIS IS THE OOM PATH, and it is now a survivable one. A killed child reports a signal here instead of // taking the parent with it. detail := runErr.Error() if s := errBuf.String(); s != "" { detail += ": " + s } log.Printf(" pdf reader: the isolated reader failed (%s), falling back", detail) return "", false } if out.Len() > workerOutputCap { log.Printf(" pdf reader: the isolated reader produced %d bytes, past the %d cap, falling back", out.Len(), workerOutputCap) return "", false } return out.String(), true }