basic pipeline from the driver project through to the database at data-connect is working.

src/hr/paycor.go

@@ -2,6 +2,11 @@ package hr
 
 import (
 	"errors"
+	"fmt"
+	"mercury/src/db"
+	"mercury/src/mercuryUtil"
+	"os"
+	"path"
 	"strconv"
 	"strings"
 )
@@ -31,6 +36,7 @@ type (
 		Sick        float64
 		Vacation    float64
 		Total       float64
+		WeekOf      string
 	}
 
 	PaycorDirectoryEntry struct {
@@ -48,6 +54,66 @@ var (
 	NewHeaders = []string{"Badge #", "Brv", "Hol", "OT", "Reg", "Service", "Sick", "Vac", "Total"}
 )
 
+func UpdatePaycorReports(pathlike string) error {
+	updateDirectory()
+	err := updatePaycorHours(pathlike)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func updateDirectory() {
+	runner := db.NewRunner("create-mercury-hrPaycorDir-table.sql", db.MercuryDatabaseName)
+	cx := &db.ConnectorGeneric{}
+	cx.ExecuteSqlScript(runner)
+
+	items := loadDirectory()
+	db.BlockUpdate[PaycorDirectoryEntry](cx, db.MercuryDatabaseName, "update-mercury-hrPaycorDirectory.sql", items)
+}
+
+func updatePaycorHours(pathlike string) error {
+	files, err := os.ReadDir(pathlike)
+	cx := &db.ConnectorGeneric{}
+	tableRunner := db.NewRunner("create-mercury-hrPaycorHours-table.sql", db.MercuryDatabaseName)
+	cx.ExecuteSqlScript(tableRunner)
+
+	if err != nil {
+		return err
+	}
+	for _, v := range files {
+		filePath := path.Join(pathlike, v.Name())
+		worker := mercuryUtil.NewCsvWorker(filePath, func() *PaycorHours { return &PaycorHours{} }, true)
+		items := make([]*PaycorHours, 0, 200)
+		err := worker.Process(&items)
+		if err != nil {
+			return err
+		}
+		for _, i := range items {
+			i.WeekOf = dateFromFileName(v.Name())
+		}
+		db.BlockUpdate[PaycorHours](cx, db.MercuryDatabaseName, "update-mercury-hrPaycorHours.sql", &items)
+	}
+	return nil
+}
+
+func loadDirectory() *[]*PaycorDirectoryEntry {
+	items := make([]*PaycorDirectoryEntry, 0, 200)
+	//todo hint: I don't like how you implemented this, David.
+	hardcodedDirectoryFileLocation := "/home/dtookey/work/clarity-reporting/paycor_dir/20220914_Paycor_Employee Roster.csv"
+	worker := mercuryUtil.NewCsvWorker(
+		hardcodedDirectoryFileLocation,
+		func() *PaycorDirectoryEntry { return &PaycorDirectoryEntry{} },
+		true,
+	)
+	err := worker.Process(&items)
+	if err != nil {
+		panic(err)
+	}
+	return &items
+}
+
 func (hl *PaycorHoursLegacy) Set(header string, content string) error {
 	content = strings.ReplaceAll(content, "\u00a0", "")
 
@@ -163,6 +229,14 @@ func (h *PaycorHours) Set(header string, content string) error {
 	return nil
 }
 
+func (h PaycorHours) ToQueryBlock() string {
+	return fmt.Sprintf("(%d,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,'%s')", h.EEid, h.Bereavement, h.Holiday, h.OT, h.Regular, h.Service, h.Sick, h.Vacation, h.Total, h.WeekOf)
+}
+
+func (d PaycorDirectoryEntry) ToQueryBlock() string {
+	return fmt.Sprintf("(%d,'%s','%s','%s','%s','%s')", d.EEid, d.Paygroup, d.LastName, d.FirstName, d.DepartmentName, d.Manager)
+}
+
 func (d *PaycorDirectoryEntry) Set(header string, content string) error {
 	switch header {
 	case "Paygroup":
@@ -233,3 +307,12 @@ func parseFloat(content string) (float64, error) {
 		return f, nil
 	}
 }
+
+func dateFromFileName(filename string) string {
+	parts := strings.Split(filename, "_")
+	block := parts[0]
+	year := block[:4]
+	month := block[4:6]
+	date := block[6:]
+	return fmt.Sprintf("%s-%s-%s", year, month, date)
+}

src/sql/create-mercury-hrPaycorDir-table.sql

@@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS mercury.hr_paycor_directory;
+
+CREATE TABLE mercury.hr_paycor_directory
+(
+    EEId     INT PRIMARY KEY,
+    paygroup VARCHAR(50),
+    lname    VARCHAR(100),
+    fname    VARCHAR(100),
+    dept     VARCHAR(100),
+    manager  VARCHAR(100)
+);

src/sql/create-mercury-hrPaycorHours-table.sql

@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS mercury.hr_paycor_report;
+
+CREATE TABLE mercury.hr_paycor_report
+(
+    EEId        INT,
+    Bereavement REAL,
+    Holiday     REAL,
+    Overtime    REAL,
+    Regular     REAL,
+    Service     REAL,
+    Sick        REAL,
+    Vacation    REAL,
+    Total       REAL,
+    week_of     Date
+);

src/sql/update-mercury-hrPaycorDirectory.sql

@@ -0,0 +1,2 @@
+INSERT INTO mercury.hr_paycor_directory (EEId, paygroup, lname, fname, dept, manager)
+VALUES %s;

src/sql/update-mercury-hrPaycorHours.sql

@@ -0,0 +1,3 @@
+INSERT INTO mercury.hr_paycor_report (EEId, Bereavement, Holiday, Overtime, Regular,
+                                      Service, Sick, Vacation, Total, week_of)
+VALUES %s;

src/test.go

@@ -12,8 +12,13 @@ import (
 )
 
 func main() {
+	csvLocation := "/home/dtookey/work/clarity-reporting/paycor/"
 	start := time.Now()
-	migrateOldSheets()
+	//migrateOldSheets()
+	err := hr.UpdatePaycorReports(csvLocation)
+	if err != nil {
+		panic(err)
+	}
 	finish := time.Now()
 	fmt.Printf("Runtime: %dus\n", finish.Sub(start).Microseconds())
 }
@@ -139,3 +144,7 @@ func getEmployeeMap() (*map[string]*hr.PaycorDirectoryEntry, error) {
 	}
 	return &eeMap, nil
 }
+
+func computeEntryHash(fname string, lname string) {
+
+}