Janady13/missionmatch / cmd/import-candidates/main.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / cmd/import-candidates/main.go
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	cmd/import-candidates/main.go
Original SHA-256	d849d28b9f6508dcfa9693de15dc5d57ab0e2133b1417058231b5500623e3765
Git blob SHA	0758aebe2a84ce414b54b3d5d09ebd1e5b4a6a7c
Reading-copy SHA-256	d849d28b9f6508dcfa9693de15dc5d57ab0e2133b1417058231b5500623e3765

Original source ↗

Every recorded occurrence

These 35 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / cmd/import-candidates/main.go	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	per-tenant-routing	9bde0f9a74b89d01980fa658e5253acda7528d0c	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	collision-deconfliction	4ee5a09972608ed89f8377c6665542d5f450f0a6	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws6-retrieval	6c56e29231ec05735927fa265c97d517499dfeaf	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-a-shared-core	e0de03ac3a1ec826bd15d0e048e3b02baa23eeff	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	m5-price-to-win	41000ad30fff4129fdaf5ec84e15ba14c27d4890	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-c-slatematch	5cd60c918301b46ecd6ed3e03e20c395da0062e2	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws1-ontology-sync	b596f449a32a1016fbd403c36505cc00a1ac9ee6	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws8-teaching-loop	c823f0fb6bb2aba0f39001eb6de96f4000d9321f	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	w3-ai-staffing-recommendations	43d56387270ef4da4cabd794d97ca6a49909bc7d	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	mag-substrate	b82c7100a492dec67dfd1eda6b3f53def859be35	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws3-clearance	4b8e7ce594c068621867f9c9deed0ec64ae6c553	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-d-command-center	255e7d1d420abf5e277d0cd8c9b14f6f72b6acb9	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	phase2-eval-fairness	d361af254701975fe239927656ababb56374655e	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-e-dataflow-deploy	433815a042f9741411e60fa553fe72143e9b10db	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ruthless-debug-sweep	b7d742f8f032984be3817efffa3eb8b13b651c1f	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	time-to-fill-survival	b66e1e505863d4f17741a4b9ede73509e45bd6c6	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ai-intelligence-layer	5d5ddff70393a197c5f54a45280bb5dd42b24755	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	graphrag-foundation	b0ddeb9f633fd138d09309d1a17b7d13bc3a67f7	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-a-opportunity-intelligence	1675031db74316e100782d063e9eda1a597fe3a1	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	deploy-and-ops	bdacd9d0b4d5998d25fc74bf4007f4c3ec538c7a	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws9-bench-ingestion	f585a9ef32fa554663ce1fba9a414a54b6e49cf0	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws2-alias-relationship	e4dada4601c6eb101830f9d223a9655dfa120616	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	presentation-readiness	eb574acaa29d71ce4d0f852572e87c275c79fdb7	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	vendor-intelligence	fe9796a19c261951d3f2565206c0c1b5ac294465	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-b-screenmatch	09015496d565a172bcaa15cfe9a8e4b072b13df0	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ws7-federal-signal	6a9f6ce9cfb13345ffb47770d4960e7c2d9c17c2	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-e-commercial-stubs	4437da660539284a400ee380b9b427a300c369ee	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	ai-brain-live	aac32fc49c878cc5bca099459307611562d0bac1	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-d-missionmatch	3be46ba89edb737182075a2b320509b42dc916f8	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	phase2-connectors	e038d4c687aec16f8bd9bc4d907eea14d91f8704	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	workstream-b-sam-contract-awards	d5101cceb98e70000cebb31d3e7ac73d747d3a6d	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original
Janady13/missionmatch / cmd/import-candidates/main.go	capture-readiness	bae6af7cb2997e819174a9e9cfff5e16067a1975	Open original

Recovered source transcript

Download complete source text ↓138 lines · 5.1 KB

// Command import-candidates sources cleared IT and cyber leads from People Data Labs (the Galapagos
// preset) and upserts them into the tenant candidates table tagged external, writing a compliance row
// per record (source, provider, query, legal basis, collected at). Clearance is left unknown: absence
// is never a negative. Run with `make import-candidates` (small free tier batches). Flags let you set
// the batch size, include contact fields, and set the legal basis. Reads PDL_API_KEY plus
// MM_MIGRATOR_DATABASE_URL plus optional MM_DEMO_ORG_ID from env.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/missionworkforce/missionmatch/internal/platform/database"
	"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
	"github.com/missionworkforce/missionmatch/internal/platform/extraction"
	"github.com/missionworkforce/missionmatch/internal/platform/sourcing"
)

const defaultOrg = "11111111-1111-1111-1111-111111111111"

func main() {
	size := flag.Int("size", 10, "number of leads to source (keep small for the free tier)")
	includeContact := flag.Bool("include-contact", false, "store email and phone (off by default)")
	legalBasis := flag.String("legal-basis",
		"Legitimate interest: recruitment sourcing of publicly available professional data",
		"the lawful basis recorded in the compliance log")
	flag.Parse()

	ctx := context.Background()
	key := strings.TrimSpace(os.Getenv("PDL_API_KEY"))
	dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
	orgStr := firstNonEmpty(os.Getenv("MM_DEMO_ORG_ID"), defaultOrg)
	if key == "" {
		log.Fatal("import-candidates: PDL_API_KEY not set (server side env only)")
	}
	if dsn == "" {
		log.Fatal("import-candidates: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
	}
	orgID, err := uuid.Parse(orgStr)
	if err != nil {
		log.Fatalf("import-candidates: bad org id %q: %v", orgStr, err)
	}

	pool, err := pgxpool.New(ctx, dsn)
	if err != nil {
		log.Fatalf("import-candidates: db connect: %v", err)
	}
	defer pool.Close()
	if err := pool.Ping(ctx); err != nil {
		log.Fatalf("import-candidates: db ping: %v", err)
	}
	// Writes the candidate and its compliance row with an explicit organization_id and no
	// app.current_org_id, so the BYPASSRLS migrator role is required or RLS rejects every insert.
	if err := database.AssertMigratorRole(ctx, pool); err != nil {
		log.Fatalf("import-candidates: %v", err)
	}
	q := dbgen.New(pool)
	ext := extraction.NewDeterministic() // certs and agencies from the profile text; skills come structured

	prov := sourcing.NewPDL(key, os.Getenv("PDL_API_BASE"), *includeContact)
	queryObj := sourcing.GalapagosSearch()
	queryJSON, _ := json.Marshal(queryObj)

	log.Printf("import-candidates: provider=%q size=%d include_contact=%v org=%s", prov.Name(), *size, *includeContact, orgID)

	people, err := prov.Search(ctx, sourcing.SearchQuery{ESQuery: queryObj, Size: *size})
	if err != nil {
		if errors.Is(err, sourcing.ErrThrottled) {
			log.Printf("import-candidates: provider rate limited (429), stopping run gracefully")
			return
		}
		log.Fatalf("import-candidates: search: %v", err)
	}
	log.Printf("import-candidates: provider returned %d leads", len(people))

	upserted := 0
	for _, sp := range people {
		if err := importOne(ctx, pool, q, ext, orgID, sp, prov.Name(), string(queryJSON), *legalBasis); err != nil {
			log.Printf("import-candidates: %s: %v", sp.ExternalID, err)
			continue
		}
		upserted++
	}
	log.Printf("import-candidates: DONE, upserted %d external candidates with compliance rows", upserted)
}

// importOne resolves and upserts one sourced person (identity resolution + structured skills, certs, and
// agencies) and writes its compliance row inside a single transaction, so a lead is never persisted
// without the legal basis record that justifies holding it. Either both commit or neither does, which
// keeps the sourcing provenance promise (a compliance row per record) true even if the second write
// fails. Only a fully committed record counts toward the total.
func importOne(ctx context.Context, pool *pgxpool.Pool, base *dbgen.Queries, ext extraction.Extractor, orgID uuid.UUID, sp sourcing.SourcedPerson, provider, queryJSON, legalBasis string) error {
	tx, err := pool.Begin(ctx)
	if err != nil {
		return fmt.Errorf("begin tx: %w", err)
	}
	defer func() { _ = tx.Rollback(ctx) }() // no op after a successful Commit

	q := base.WithTx(tx)
	candID, err := sourcing.IngestPerson(ctx, q, orgID, sp, ext)
	if err != nil {
		return err
	}
	if err := q.InsertSourcingLog(ctx, dbgen.InsertSourcingLogParams{
		OrganizationID: orgID,
		CandidateID:    &candID,
		ExternalID:     sp.ExternalID,
		Source:         "PDL",
		Provider:       provider,
		Query:          queryJSON,
		LegalBasis:     legalBasis,
	}); err != nil {
		return fmt.Errorf("insert sourcing log: %w", err)
	}
	if err := tx.Commit(ctx); err != nil {
		return fmt.Errorf("commit tx: %w", err)
	}
	return nil
}

func firstNonEmpty(vals ...string) string {
	for _, v := range vals {
		if s := strings.TrimSpace(v); s != "" {
			return s
		}
	}
	return ""
}