// Command import-candidates sources cleared IT and cyber leads from People Data Labs (the Galapagos // preset) and upserts them into the tenant candidates table tagged external, writing a compliance row // per record (source, provider, query, legal basis, collected at). Clearance is left unknown: absence // is never a negative. Run with `make import-candidates` (small free tier batches). Flags let you set // the batch size, include contact fields, and set the legal basis. Reads PDL_API_KEY plus // MM_MIGRATOR_DATABASE_URL plus optional MM_DEMO_ORG_ID from env. package main import ( "context" "encoding/json" "errors" "flag" "log" "os" "strings" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" "github.com/missionworkforce/missionmatch/internal/platform/database/dbgen" "github.com/missionworkforce/missionmatch/internal/platform/sourcing" ) const defaultOrg = "11111111-1111-1111-1111-111111111111" func main() { size := flag.Int("size", 10, "number of leads to source (keep small for the free tier)") includeContact := flag.Bool("include-contact", false, "store email and phone (off by default)") legalBasis := flag.String("legal-basis", "Legitimate interest: recruitment sourcing of publicly available professional data", "the lawful basis recorded in the compliance log") flag.Parse() ctx := context.Background() key := strings.TrimSpace(os.Getenv("PDL_API_KEY")) dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL")) orgStr := firstNonEmpty(os.Getenv("MM_DEMO_ORG_ID"), defaultOrg) if key == "" { log.Fatal("import-candidates: PDL_API_KEY not set (server side env only)") } if dsn == "" { log.Fatal("import-candidates: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)") } orgID, err := uuid.Parse(orgStr) if err != nil { log.Fatalf("import-candidates: bad org id %q: %v", orgStr, err) } pool, err := pgxpool.New(ctx, dsn) if err != nil { log.Fatalf("import-candidates: db connect: %v", err) } defer pool.Close() if err := pool.Ping(ctx); err != nil { log.Fatalf("import-candidates: db ping: %v", err) } q := dbgen.New(pool) prov := sourcing.NewPDL(key, os.Getenv("PDL_API_BASE"), *includeContact) queryObj := sourcing.GalapagosSearch() queryJSON, _ := json.Marshal(queryObj) log.Printf("import-candidates: provider=%q size=%d include_contact=%v org=%s", prov.Name(), *size, *includeContact, orgID) people, err := prov.Search(ctx, sourcing.SearchQuery{ESQuery: queryObj, Size: *size}) if err != nil { if errors.Is(err, sourcing.ErrThrottled) { log.Printf("import-candidates: provider rate limited (429), stopping run gracefully") return } log.Fatalf("import-candidates: search: %v", err) } log.Printf("import-candidates: provider returned %d leads", len(people)) upserted := 0 for _, sp := range people { headline := buildHeadline(sp) extID := sp.ExternalID var email *string if len(sp.Emails) > 0 { email = &sp.Emails[0] } loc := sp.Location candID, err := q.UpsertExternalCandidate(ctx, dbgen.UpsertExternalCandidateParams{ OrganizationID: orgID, FullName: sp.FullName, Email: email, Headline: strPtr(headline), YearsExperience: int32(sp.Years), Location: strPtr(loc), Remote: strings.Contains(strings.ToLower(loc), "remote"), ExternalID: &extID, }) if err != nil { log.Printf("import-candidates: upsert %s: %v", sp.ExternalID, err) continue } if err := q.InsertSourcingLog(ctx, dbgen.InsertSourcingLogParams{ OrganizationID: orgID, CandidateID: &candID, ExternalID: sp.ExternalID, Source: "PDL", Provider: prov.Name(), Query: string(queryJSON), LegalBasis: *legalBasis, }); err != nil { log.Printf("import-candidates: log %s: %v", sp.ExternalID, err) } upserted++ } log.Printf("import-candidates: DONE, upserted %d external candidates with compliance rows", upserted) } func buildHeadline(sp sourcing.SourcedPerson) string { skills := sp.Skills if len(skills) > 6 { skills = skills[:6] } h := sp.JobTitle if h == "" { h = sp.Role } if len(skills) > 0 { if h != "" { h += " | " } h += strings.Join(skills, ", ") } return strings.TrimSpace(h) } func strPtr(s string) *string { if strings.TrimSpace(s) == "" { return nil } return &s } func firstNonEmpty(vals ...string) string { for _, v := range vals { if s := strings.TrimSpace(v); s != "" { return s } } return "" }