Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/missionmatch / cmd/import-candidates/main.go |
| Date | Not specified in source |
| Repository | Janady13/missionmatch |
| Revision | 39df94f92b2a33f852481fe1954dabed841524bb |
| Branch | w0-aedt-compliance |
| Path or archive member | cmd/import-candidates/main.go |
| Original SHA-256 | bd8739261454c038fbefeab1fc3281c96835b17136002bfa5829cf8abc02f8d4 |
| Git blob SHA | e8f6b9f1d755abd7ef12f05c30a02961e415bf9c |
| Reading-copy SHA-256 | bd8739261454c038fbefeab1fc3281c96835b17136002bfa5829cf8abc02f8d4 |
Recovered source transcript
Download complete source text ↓148 lines · 4.4 KB
// Command import-candidates sources cleared IT and cyber leads from People Data Labs (the Galapagos
// preset) and upserts them into the tenant candidates table tagged external, writing a compliance row
// per record (source, provider, query, legal basis, collected at). Clearance is left unknown: absence
// is never a negative. Run with `make import-candidates` (small free tier batches). Flags let you set
// the batch size, include contact fields, and set the legal basis. Reads PDL_API_KEY plus
// MM_MIGRATOR_DATABASE_URL plus optional MM_DEMO_ORG_ID from env.
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"log"
"os"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/missionworkforce/missionmatch/internal/platform/database/dbgen"
"github.com/missionworkforce/missionmatch/internal/platform/sourcing"
)
const defaultOrg = "11111111-1111-1111-1111-111111111111"
func main() {
size := flag.Int("size", 10, "number of leads to source (keep small for the free tier)")
includeContact := flag.Bool("include-contact", false, "store email and phone (off by default)")
legalBasis := flag.String("legal-basis",
"Legitimate interest: recruitment sourcing of publicly available professional data",
"the lawful basis recorded in the compliance log")
flag.Parse()
ctx := context.Background()
key := strings.TrimSpace(os.Getenv("PDL_API_KEY"))
dsn := firstNonEmpty(os.Getenv("MM_MIGRATOR_DATABASE_URL"), os.Getenv("MM_DATABASE_URL"))
orgStr := firstNonEmpty(os.Getenv("MM_DEMO_ORG_ID"), defaultOrg)
if key == "" {
log.Fatal("import-candidates: PDL_API_KEY not set (server side env only)")
}
if dsn == "" {
log.Fatal("import-candidates: set MM_MIGRATOR_DATABASE_URL (or MM_DATABASE_URL)")
}
orgID, err := uuid.Parse(orgStr)
if err != nil {
log.Fatalf("import-candidates: bad org id %q: %v", orgStr, err)
}
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
log.Fatalf("import-candidates: db connect: %v", err)
}
defer pool.Close()
if err := pool.Ping(ctx); err != nil {
log.Fatalf("import-candidates: db ping: %v", err)
}
q := dbgen.New(pool)
prov := sourcing.NewPDL(key, os.Getenv("PDL_API_BASE"), *includeContact)
queryObj := sourcing.GalapagosSearch()
queryJSON, _ := json.Marshal(queryObj)
log.Printf("import-candidates: provider=%q size=%d include_contact=%v org=%s", prov.Name(), *size, *includeContact, orgID)
people, err := prov.Search(ctx, sourcing.SearchQuery{ESQuery: queryObj, Size: *size})
if err != nil {
if errors.Is(err, sourcing.ErrThrottled) {
log.Printf("import-candidates: provider rate limited (429), stopping run gracefully")
return
}
log.Fatalf("import-candidates: search: %v", err)
}
log.Printf("import-candidates: provider returned %d leads", len(people))
upserted := 0
for _, sp := range people {
headline := buildHeadline(sp)
extID := sp.ExternalID
var email *string
if len(sp.Emails) > 0 {
email = &sp.Emails[0]
}
loc := sp.Location
candID, err := q.UpsertExternalCandidate(ctx, dbgen.UpsertExternalCandidateParams{
OrganizationID: orgID,
FullName: sp.FullName,
Email: email,
Headline: strPtr(headline),
YearsExperience: int32(sp.Years),
Location: strPtr(loc),
Remote: strings.Contains(strings.ToLower(loc), "remote"),
ExternalID: &extID,
})
if err != nil {
log.Printf("import-candidates: upsert %s: %v", sp.ExternalID, err)
continue
}
if err := q.InsertSourcingLog(ctx, dbgen.InsertSourcingLogParams{
OrganizationID: orgID,
CandidateID: &candID,
ExternalID: sp.ExternalID,
Source: "PDL",
Provider: prov.Name(),
Query: string(queryJSON),
LegalBasis: *legalBasis,
}); err != nil {
log.Printf("import-candidates: log %s: %v", sp.ExternalID, err)
}
upserted++
}
log.Printf("import-candidates: DONE, upserted %d external candidates with compliance rows", upserted)
}
func buildHeadline(sp sourcing.SourcedPerson) string {
skills := sp.Skills
if len(skills) > 6 {
skills = skills[:6]
}
h := sp.JobTitle
if h == "" {
h = sp.Role
}
if len(skills) > 0 {
if h != "" {
h += " | "
}
h += strings.Join(skills, ", ")
}
return strings.TrimSpace(h)
}
func strPtr(s string) *string {
if strings.TrimSpace(s) == "" {
return nil
}
return &s
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if s := strings.TrimSpace(v); s != "" {
return s
}
}
return ""
}