Janady13/missionmatch / internal/platform/sourcing/ingest.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / internal/platform/sourcing/ingest.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	09015496d565a172bcaa15cfe9a8e4b072b13df0
Branch	workstream-b-screenmatch
Path or archive member	internal/platform/sourcing/ingest.go
Original SHA-256	fa33ea6a5ce099d54017158ed503c46af36cfbd9581eda2bbbc1eee1799ac4dc
Git blob SHA	06384ae2c2f827b3902f58210c840b4fd1b9f5ce
Reading-copy SHA-256	fa33ea6a5ce099d54017158ed503c46af36cfbd9581eda2bbbc1eee1799ac4dc

Original source ↗

Every recorded occurrence

These 43 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / internal/platform/sourcing/ingest.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original

Recovered source transcript

Download complete source text ↓167 lines · 5.9 KB

package sourcing

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v5"

	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/extraction"
)

// IngestPerson resolves a sourced person to a candidate record and writes their STRUCTURED workforce
// data, so a sourced lead is a real scoring input (skills, certs, customer agencies as rows), not a free
// text headline, and never a duplicate of an existing bench person. It runs on the caller's transaction;
// the import command wraps it with the sourcing_log (consent and legal basis) write. Returns the
// resolved candidate id.
//
// Identity resolution uses blocking plus multi attribute matching: an exact email match merges into the
// existing person; failing that, an exact normalized name plus location match merges; otherwise a new
// external candidate is created. A merge ENRICHES (links the external id, backfills an empty headline and
// a lower years figure, adds any new skills, certs, and agencies) and never clobbers curated bench data.
func IngestPerson(ctx context.Context, q *dbgen.Queries, orgID uuid.UUID, sp SourcedPerson, ext extraction.Extractor) (uuid.UUID, error) {
	candID, found, err := resolveCandidate(ctx, q, orgID, sp)
	if err != nil {
		return uuid.Nil, err
	}

	if found {
		if err := q.LinkSourcedCandidate(ctx, dbgen.LinkSourcedCandidateParams{
			ID: candID, ExternalID: nzPtr(sp.ExternalID), Headline: nzPtr(buildHeadline(sp)), YearsExperience: int32(sp.Years),
		}); err != nil {
			return uuid.Nil, fmt.Errorf("link sourced candidate: %w", err)
		}
	} else {
		extID := sp.ExternalID
		candID, err = q.UpsertExternalCandidate(ctx, dbgen.UpsertExternalCandidateParams{
			OrganizationID:  orgID,
			FullName:        sp.FullName,
			Email:           firstOrNil(sp.Emails),
			Headline:        nzPtr(buildHeadline(sp)),
			YearsExperience: int32(sp.Years),
			Location:        nzPtr(sp.Location),
			Remote:          strings.Contains(strings.ToLower(sp.Location), "remote"),
			ExternalID:      &extID,
		})
		if err != nil {
			return uuid.Nil, fmt.Errorf("upsert external candidate: %w", err)
		}
	}

	// Skills come from the provider's STRUCTURED field, more reliable than gazetteer extraction. Each is
	// upserted into the ontology then attached; the add queries are idempotent (ON CONFLICT DO NOTHING).
	for _, name := range dedupe(sp.Skills) {
		sk, err := q.UpsertSkill(ctx, dbgen.UpsertSkillParams{Name: name})
		if err != nil {
			return uuid.Nil, fmt.Errorf("upsert skill: %w", err)
		}
		if err := q.AddCandidateSkill(ctx, dbgen.AddCandidateSkillParams{OrganizationID: orgID, CandidateID: candID, SkillID: sk.ID}); err != nil {
			return uuid.Nil, fmt.Errorf("add candidate skill: %w", err)
		}
	}

	// Certifications and customer agencies are not structured in the provider feed, so extract them from
	// the profile text via the extraction port. Its skills output is ignored in favor of sp.Skills.
	rx, err := ext.ExtractResume(ctx, profileText(sp))
	if err != nil {
		return uuid.Nil, fmt.Errorf("extract profile: %w", err)
	}
	for _, name := range dedupe(rx.Certifications) {
		if err := q.AddCandidateCertification(ctx, dbgen.AddCandidateCertificationParams{OrganizationID: orgID, CandidateID: candID, Name: name}); err != nil {
			return uuid.Nil, fmt.Errorf("add candidate certification: %w", err)
		}
	}
	for _, agency := range dedupe(rx.Agencies) {
		if err := q.AddCandidateCustomerExperience(ctx, dbgen.AddCandidateCustomerExperienceParams{OrganizationID: orgID, CandidateID: candID, Agency: agency}); err != nil {
			return uuid.Nil, fmt.Errorf("add candidate customer experience: %w", err)
		}
	}
	return candID, nil
}

// resolveCandidate returns an existing candidate id for the sourced person when identity resolution
// matches, else found is false. Email is tried first (strongest signal), then name plus a non empty
// location. ErrNoRows is the normal not found path, not an error.
func resolveCandidate(ctx context.Context, q *dbgen.Queries, orgID uuid.UUID, sp SourcedPerson) (id uuid.UUID, found bool, err error) {
	if len(sp.Emails) > 0 && strings.TrimSpace(sp.Emails[0]) != "" {
		got, e := q.FindCandidateByEmail(ctx, dbgen.FindCandidateByEmailParams{OrganizationID: orgID, Lower: sp.Emails[0]})
		if e == nil {
			return got, true, nil
		}
		if !errors.Is(e, pgx.ErrNoRows) {
			return uuid.Nil, false, fmt.Errorf("find by email: %w", e)
		}
	}
	if strings.TrimSpace(sp.FullName) != "" && strings.TrimSpace(sp.Location) != "" {
		got, e := q.FindCandidateByNameLocation(ctx, dbgen.FindCandidateByNameLocationParams{OrganizationID: orgID, Lower: sp.FullName, Lower_2: sp.Location})
		if e == nil {
			return got, true, nil
		}
		if !errors.Is(e, pgx.ErrNoRows) {
			return uuid.Nil, false, fmt.Errorf("find by name and location: %w", e)
		}
	}
	return uuid.Nil, false, nil
}

// buildHeadline composes a self reported title and skills line for a sourced lead.
func buildHeadline(sp SourcedPerson) string {
	skills := sp.Skills
	if len(skills) > 6 {
		skills = skills[:6]
	}
	h := sp.JobTitle
	if h == "" {
		h = sp.Role
	}
	if len(skills) > 0 {
		if h != "" {
			h += " | "
		}
		h += strings.Join(skills, ", ")
	}
	return strings.TrimSpace(h)
}

// profileText is the free text the extractor reads for certs and agencies (skills come structured).
func profileText(sp SourcedPerson) string {
	return strings.Join([]string{buildHeadline(sp), strings.Join(sp.Skills, " "), sp.JobTitle, sp.Role}, " ")
}

func nzPtr(s string) *string {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	return &s
}

func firstOrNil(xs []string) *string {
	if len(xs) > 0 {
		return nzPtr(xs[0])
	}
	return nil
}

func dedupe(xs []string) []string {
	seen := map[string]bool{}
	out := make([]string, 0, len(xs))
	for _, x := range xs {
		t := strings.TrimSpace(x)
		if t == "" {
			continue
		}
		k := strings.ToLower(t)
		if seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, t)
	}
	return out
}