package sourcing import ( "context" "errors" "fmt" "strings" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" "github.com/missionworkforce/missionmatch/internal/platform/extraction" ) // IngestPerson resolves a sourced person to a candidate record and writes their STRUCTURED workforce // data, so a sourced lead is a real scoring input (skills, certs, customer agencies as rows), not a free // text headline, and never a duplicate of an existing bench person. It runs on the caller's transaction; // the import command wraps it with the sourcing_log (consent and legal basis) write. Returns the // resolved candidate id. // // Identity resolution uses blocking plus multi attribute matching: an exact email match merges into the // existing person; failing that, an exact normalized name plus location match merges; otherwise a new // external candidate is created. A merge ENRICHES (links the external id, backfills an empty headline and // a lower years figure, adds any new skills, certs, and agencies) and never clobbers curated bench data. func IngestPerson(ctx context.Context, q *dbgen.Queries, orgID uuid.UUID, sp SourcedPerson, ext extraction.Extractor) (uuid.UUID, error) { candID, _, err := writeStructuredCandidate(ctx, q, orgID, sp, ext) return candID, err } // writeStructuredCandidate is the shared body of the sourced lead and the bench record ingest paths. It // resolves identity (email then name plus location blocking), upserts or enriches the candidate THROUGH // the same clearance hardcoding path (UpsertExternalCandidate writes clearance_level 'unknown', // clearance_poly 'none', clearance_status 'unknown', so a sourced or bench clearance code can never score // as a held clearance), and writes the structured skills, certs, and agencies verbatim. It returns the // resolved candidate id and whether the row was an existing person (an enrich) or a new insert. Both the // sourced lead path and the bench record path call this, so the dedup and the structured writes are // byte identical between them and IngestPerson's signature is unchanged for its existing caller. func writeStructuredCandidate(ctx context.Context, q *dbgen.Queries, orgID uuid.UUID, sp SourcedPerson, ext extraction.Extractor) (candID uuid.UUID, found bool, err error) { candID, found, err = resolveCandidate(ctx, q, orgID, sp) if err != nil { return uuid.Nil, false, err } if found { if err := q.LinkSourcedCandidate(ctx, dbgen.LinkSourcedCandidateParams{ ID: candID, ExternalID: nzPtr(sp.ExternalID), Headline: nzPtr(buildHeadline(sp)), YearsExperience: int32(sp.Years), }); err != nil { return uuid.Nil, false, fmt.Errorf("link sourced candidate: %w", err) } } else { // ExternalID is NULL when empty (nzPtr), never the empty string. The candidates partial unique index // is (organization_id, external_id) WHERE external_id IS NOT NULL, so an empty string would collapse // every external id free bench record onto one (org, '') row. A bench file usually carries no external // id, so identity for those records is resolved only by the email then name plus location dedup above, // and a genuinely new person inserts a fresh row. candID, err = q.UpsertExternalCandidate(ctx, dbgen.UpsertExternalCandidateParams{ OrganizationID: orgID, FullName: sp.FullName, Email: firstOrNil(sp.Emails), Headline: nzPtr(buildHeadline(sp)), YearsExperience: int32(sp.Years), Location: nzPtr(sp.Location), Remote: strings.Contains(strings.ToLower(sp.Location), "remote"), ExternalID: nzPtr(sp.ExternalID), }) if err != nil { return uuid.Nil, false, fmt.Errorf("upsert external candidate: %w", err) } } // Skills come from the provider's STRUCTURED field, more reliable than gazetteer extraction. Each is // upserted into the ontology then attached; the add queries are idempotent (ON CONFLICT DO NOTHING). for _, name := range dedupe(sp.Skills) { sk, err := q.UpsertSkill(ctx, dbgen.UpsertSkillParams{Name: name}) if err != nil { return uuid.Nil, false, fmt.Errorf("upsert skill: %w", err) } if err := q.AddCandidateSkill(ctx, dbgen.AddCandidateSkillParams{OrganizationID: orgID, CandidateID: candID, SkillID: sk.ID}); err != nil { return uuid.Nil, false, fmt.Errorf("add candidate skill: %w", err) } } // Certifications and customer agencies are not structured in the provider feed, so extract them from // the profile text via the extraction port. Its skills output is ignored in favor of sp.Skills, and its // ClearanceLevel output is intentionally ignored too, so a resume regex never reaches a clearance column. rx, err := ext.ExtractResume(ctx, profileText(sp)) if err != nil { return uuid.Nil, false, fmt.Errorf("extract profile: %w", err) } for _, name := range dedupe(rx.Certifications) { if err := q.AddCandidateCertification(ctx, dbgen.AddCandidateCertificationParams{OrganizationID: orgID, CandidateID: candID, Name: name}); err != nil { return uuid.Nil, false, fmt.Errorf("add candidate certification: %w", err) } } for _, agency := range dedupe(rx.Agencies) { if err := q.AddCandidateCustomerExperience(ctx, dbgen.AddCandidateCustomerExperienceParams{OrganizationID: orgID, CandidateID: candID, Agency: agency}); err != nil { return uuid.Nil, false, fmt.Errorf("add candidate customer experience: %w", err) } } return candID, found, nil } // resolveCandidate returns an existing candidate id for the sourced person when identity resolution // matches, else found is false. Email is tried first (strongest signal), then name plus a non empty // location. ErrNoRows is the normal not found path, not an error. func resolveCandidate(ctx context.Context, q *dbgen.Queries, orgID uuid.UUID, sp SourcedPerson) (id uuid.UUID, found bool, err error) { if len(sp.Emails) > 0 && strings.TrimSpace(sp.Emails[0]) != "" { got, e := q.FindCandidateByEmail(ctx, dbgen.FindCandidateByEmailParams{OrganizationID: orgID, Lower: sp.Emails[0]}) if e == nil { return got, true, nil } if !errors.Is(e, pgx.ErrNoRows) { return uuid.Nil, false, fmt.Errorf("find by email: %w", e) } } if strings.TrimSpace(sp.FullName) != "" && strings.TrimSpace(sp.Location) != "" { got, e := q.FindCandidateByNameLocation(ctx, dbgen.FindCandidateByNameLocationParams{OrganizationID: orgID, Lower: sp.FullName, Lower_2: sp.Location}) if e == nil { return got, true, nil } if !errors.Is(e, pgx.ErrNoRows) { return uuid.Nil, false, fmt.Errorf("find by name and location: %w", e) } } return uuid.Nil, false, nil } // buildHeadline composes a self reported title and skills line for a sourced lead. func buildHeadline(sp SourcedPerson) string { skills := sp.Skills if len(skills) > 6 { skills = skills[:6] } h := sp.JobTitle if h == "" { h = sp.Role } if len(skills) > 0 { if h != "" { h += " | " } h += strings.Join(skills, ", ") } return strings.TrimSpace(h) } // profileText is the free text the extractor reads for certs and agencies (skills come structured). func profileText(sp SourcedPerson) string { return strings.Join([]string{buildHeadline(sp), strings.Join(sp.Skills, " "), sp.JobTitle, sp.Role}, " ") } func nzPtr(s string) *string { s = strings.TrimSpace(s) if s == "" { return nil } return &s } func firstOrNil(xs []string) *string { if len(xs) > 0 { return nzPtr(xs[0]) } return nil } func dedupe(xs []string) []string { seen := map[string]bool{} out := make([]string, 0, len(xs)) for _, x := range xs { t := strings.TrimSpace(x) if t == "" { continue } k := strings.ToLower(t) if seen[k] { continue } seen[k] = true out = append(out, t) } return out }