From e40a72f38eae71592051d616350c3478d5842cf3 Mon Sep 17 00:00:00 2001 From: dtookey Date: Wed, 4 May 2022 11:51:20 -0400 Subject: [PATCH] refactoring of the database is complete and a lot more clean. It is currently untested, so we'll kick off the inaugural scan here in a minute. --- go.mod | 8 +- src/clarity/clarity-database.go | 1 + src/clarity/s3connect.go | 32 +++ src/clarity/snitch.go | 41 ++++ src/insight/insight-database.go | 194 ------------------ src/mercury.go | 22 +- src/mercury/Interconnect.go | 68 ++++++ .../insight-connect.go | 92 ++------- src/projectInsight/insight-database.go | 107 ++++++++++ ...sql => create-insight-timeentry-table.sql} | 0 src/sql/read-clarity-allProjects.sql | 1 + src/sql/read-insight-allUsers.sql | 1 + src/sql/read-insight-engineerUsers.sql | 4 + ...entry.sql => update-insight-timeEntry.sql} | 0 ...ate-users.sql => update-insight-users.sql} | 0 src/util/database-primitives.go | 130 ++++++++++++ 16 files changed, 424 insertions(+), 277 deletions(-) create mode 100644 src/clarity/clarity-database.go create mode 100644 src/clarity/s3connect.go create mode 100644 src/clarity/snitch.go delete mode 100644 src/insight/insight-database.go create mode 100644 src/mercury/Interconnect.go rename src/{insight => projectInsight}/insight-connect.go (75%) create mode 100644 src/projectInsight/insight-database.go rename src/sql/{create-timeentry-table.sql => create-insight-timeentry-table.sql} (100%) create mode 100644 src/sql/read-clarity-allProjects.sql create mode 100644 src/sql/read-insight-allUsers.sql create mode 100644 src/sql/read-insight-engineerUsers.sql rename src/sql/{update-timeentry.sql => update-insight-timeEntry.sql} (100%) rename src/sql/{update-users.sql => update-insight-users.sql} (100%) create mode 100644 src/util/database-primitives.go diff --git a/go.mod b/go.mod index b836343..9c78da9 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module mercury go 1.18 -require ( - github.com/go-sql-driver/mysql v1.6.0 -) \ No newline at end of file +require github.com/go-sql-driver/mysql v1.6.0 + +require github.com/aws/aws-sdk-go v1.44.6 + +require github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/src/clarity/clarity-database.go b/src/clarity/clarity-database.go new file mode 100644 index 0000000..059e100 --- /dev/null +++ b/src/clarity/clarity-database.go @@ -0,0 +1 @@ +package clarity diff --git a/src/clarity/s3connect.go b/src/clarity/s3connect.go new file mode 100644 index 0000000..d7d7f7d --- /dev/null +++ b/src/clarity/s3connect.go @@ -0,0 +1,32 @@ +package clarity + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "log" +) + +func GetS3ListingForKey(region string, bucket string, key string) *[]*s3.Object { + ses, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + + if err != nil { + log.Panicln(err) + } + + conn := s3.New(ses) + + req := s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(key), + } + + resp, err := conn.ListObjectsV2(&req) + if err != nil { + log.Panic(err) + } + + return &resp.Contents +} diff --git a/src/clarity/snitch.go b/src/clarity/snitch.go new file mode 100644 index 0000000..92e956d --- /dev/null +++ b/src/clarity/snitch.go @@ -0,0 +1,41 @@ +package clarity + +import ( + "log" + "mercury/src/projectInsight" +) + +// +/*====================================================================================== + Snitch +======================================================================================*/ + +type Snitch struct { + DB projectInsight.InsightDBConnector +} + +//func NewSnitch() *Snitch { +// return &Snitch{} +//} + +func (s *Snitch) Test() { + + res := GetS3ListingForKey("us-east-1", "jds.private.rdu.str", "RDU2201010/") + + for _, thing := range *res { + log.Println(*thing.Key) + } +} + +func (s *Snitch) getProjectNumbersForTesting() *[]*string { + return nil +} + +// + +// +/*====================================================================================== + utility functions +======================================================================================*/ + +// diff --git a/src/insight/insight-database.go b/src/insight/insight-database.go deleted file mode 100644 index e7eb6f5..0000000 --- a/src/insight/insight-database.go +++ /dev/null @@ -1,194 +0,0 @@ -package insight - -import ( - "database/sql" - "fmt" - _ "github.com/go-sql-driver/mysql" - "io/ioutil" - "log" - "os" - "strings" -) - -const ( - InsightDatabaseName = "insight" - dbCredsEnvName = "DB_CREDS" - dbCredsHostName = "DB_HOST" -) - -// -/*====================================================================================== - DBConnector -======================================================================================*/ - -type DBConnector struct{} - -func NewDBConnection() *DBConnector { - return &DBConnector{} -} - -func (c *DBConnector) CreateTables() { - tableCreationScripts := []string{"create-insight-user-table.sql", "create-timeentry-table.sql"} - - for _, scriptName := range tableCreationScripts { - c.ExecuteSqlScript(InsightDatabaseName, scriptName) - } -} - -func (c *DBConnector) UpdateTimeEntries(entries *[]*TimeEntry) { - statement := loadSqlFile("update-timeentry.sql") - db := c.checkoutConnection(InsightDatabaseName) - defer c.returnConnection(db) - s, err := db.Prepare(*statement) - defer s.Close() - if err != nil { - log.Panic(err) - } - for _, ent := range *entries { - _, err = s.Exec(ent.ActualHours, - ent.TimeEntryDate, - ent.TimeEntryDescription, - ent.ProjectId, - ent.TaskId, - ent.TimeSheetId, - ent.UserId) - if err != nil { - log.Printf("%#v\n", s) - log.Panic(err) - } - } -} - -func (c *DBConnector) UpdateUsers(users *[]User) { - statement := loadSqlFile("update-users.sql") - db := c.checkoutConnection(InsightDatabaseName) - defer c.returnConnection(db) - s, err := db.Prepare(*statement) - defer s.Close() - if err != nil { - log.Panic(err) - } - for _, user := range *users { - _, err = s.Exec(user.Id, user.FirstName, user.LastName, user.EmailAddress) - if err != nil { - log.Printf("%#v\n", s) - log.Panic(err) - } - } -} - -func (c *DBConnector) FetchUsers() *[]*User { - ret := make([]*User, 0, 50) - cx := c.checkoutConnection(InsightDatabaseName) - queryText := "SELECT * FROM users;" - rs, err := cx.Query(queryText) - if err != nil { - log.Panic(err) - } - for rs.Next() { - u := User{} - err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress) - if err != nil { - log.Panic(err) - } - ret = append(ret, &u) - } - return &ret -} - -func (c *DBConnector) FetchEngineerUsers() *[]*User { - ret := make([]*User, 0, 50) - cx := c.checkoutConnection(InsightDatabaseName) - queryText := "SELECT insight.users.Id, insight.users.FirstName, insight.users.LastName, insight.users.EmailAddress FROM insight.users INNER JOIN projects.users on insight.users.EmailAddress = projects.users.email where projects.users.priv & POW(2, 25) > 0;" - rs, err := cx.Query(queryText) - if err != nil { - log.Panic(err) - } - for rs.Next() { - u := User{} - err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress) - if err != nil { - log.Panic(err) - } - ret = append(ret, &u) - } - return &ret -} - -func (c *DBConnector) ExecuteSqlScript(database string, scriptName string) { - db := c.checkoutConnection(database) - defer c.returnConnection(db) - queryWhole := *loadSqlFile(scriptName) - queryParts := strings.Split(queryWhole, ";") - - log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole) - - for _, query := range queryParts { - if len(query) == 0 || query == "\n" { - continue - } - _, err := db.Exec(query + ";") - if err != nil { - log.Printf("Query involved in error: %s\n", query) - log.Panic(err) - } - } -} - -func (c *DBConnector) checkoutConnection(dataBase string) *sql.DB { - return createDbConnection(dataBase) -} - -func (c *DBConnector) returnConnection(db *sql.DB) { - db.Close() -} - -// - -// -/*====================================================================================== - Utility Functions -======================================================================================*/ - -func createDbConnection(database string) *sql.DB { - cred := os.Getenv(dbCredsEnvName) - host := os.Getenv(dbCredsHostName) - dbString := "clarity:%s@tcp(%s)/%s" - connectString := fmt.Sprintf(dbString, cred, host, database) - - db, err := sql.Open("mysql", connectString) - if err != nil { - log.Panic(err) - } - return db -} - -func getSecret(pathlike string) *string { - file, err := os.OpenFile(pathlike, os.O_RDONLY, 0755) - if err != nil { - log.Panic(err) - } - defer file.Close() - raw, err := ioutil.ReadAll(file) - if err != nil { - log.Panic(err) - } - str := strings.Trim(string(raw), "\n") - return &str -} - -func loadSqlFile(scriptName string) *string { - file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755) - if err != nil { - log.Panic(err) - } - defer file.Close() - raw, err := ioutil.ReadAll(file) - if err != nil { - log.Panic(err) - } - str := strings.Trim(string(raw), "\n") - return &str -} - -// diff --git a/src/mercury.go b/src/mercury.go index 7d0b146..29ec7e8 100644 --- a/src/mercury.go +++ b/src/mercury.go @@ -2,8 +2,8 @@ package main import ( "log" - "mercury/src/insight" "mercury/src/mercury" + "mercury/src/projectInsight" "os" "time" ) @@ -12,18 +12,28 @@ func main() { s := time.Now() //icx := insight.NewInsightConnect() - //icx.DBConnector.ExecuteSqlScript("insight", "create-insight-contribution-table.sql") + //icx.InsightDBConnector.ExecuteSqlScript("insight", "create-insight-contribution-table.sql") //processQbBilling() - test() - //fetchInsightData() + updateInsightData() + //test() f := time.Now() log.Println(f.Sub(s).Microseconds()) } func test() { - icx := insight.NewInsightConnect() + //clarity.NewSnitch().Test() + insightThing := projectInsight.NewDBConnection() + res := insightThing.ReadEngineerUsers() + for _, usr := range *res { + log.Println(usr.EmailAddress) + } + +} + +func updateInsightData() { + icx := mercury.NewInterconnect() icx.ResetTables() icx.UpdateUsers() icx.UpdateTimeEntries() @@ -36,5 +46,5 @@ func processQbBilling() { log.Fatalln("please set the mercury_path env var. we don't know where to look otherwise") } - mercury.ProcessTrialBalances(reportBase, "./test.csv") + mercury.ProcessTrialBalances(reportBase, "./updateInsightData.csv") } diff --git a/src/mercury/Interconnect.go b/src/mercury/Interconnect.go new file mode 100644 index 0000000..69dbbd1 --- /dev/null +++ b/src/mercury/Interconnect.go @@ -0,0 +1,68 @@ +package mercury + +import ( + "log" + "mercury/src/projectInsight" + "time" +) + +type Interconnect struct { + Client *projectInsight.InsightClient + InsightDBConnector *projectInsight.InsightDBConnector +} + +// +/*====================================================================================== + interconnect +======================================================================================*/ + +func NewInterconnect() *Interconnect { + connect := Interconnect{} + connect.Client = projectInsight.NewIClient() + connect.InsightDBConnector = &projectInsight.InsightDBConnector{} + return &connect +} + +func (ic *Interconnect) ResetTables() { + ic.InsightDBConnector.CreateTables() +} + +func (ic *Interconnect) UpdateUsers() { + users := ic.Client.GetUsers() + ic.InsightDBConnector.UpdateUsers(users) +} + +func (ic *Interconnect) UpdateTimeEntries() { + users := ic.InsightDBConnector.ReadUsers() + entryChan := make(chan *[]*projectInsight.InsightTimeEntry) + coroutineCount := 0 + dateString := createDateString() + for _, userPtr := range *users { + go ic.Client.GetTimeAllTimeEntriesForUserThroughDate(userPtr.Id, dateString, entryChan) + coroutineCount++ + time.Sleep(1000 * time.Millisecond) + } + + log.Printf("Currently pending goroutines: %d\n", coroutineCount) + for coroutineCount > 0 { + entries := <-entryChan + ic.InsightDBConnector.UpdateTimeEntries(entries) + coroutineCount-- + log.Printf("Currently pending goroutines: %d\n", coroutineCount) + } + ic.InsightDBConnector.ExecuteSqlScript("insight", "create-insight-contribution-table.sql") +} + +// + +// +/*====================================================================================== + UtilityFunctions +======================================================================================*/ + +func createDateString() string { + now := time.Now() + return now.Format("2006-01-02") +} + +// diff --git a/src/insight/insight-connect.go b/src/projectInsight/insight-connect.go similarity index 75% rename from src/insight/insight-connect.go rename to src/projectInsight/insight-connect.go index c51a684..99be0c3 100644 --- a/src/insight/insight-connect.go +++ b/src/projectInsight/insight-connect.go @@ -1,4 +1,4 @@ -package insight +package projectInsight import ( "encoding/json" @@ -8,7 +8,6 @@ import ( "net/http" "os" "strings" - "time" ) const ( @@ -16,24 +15,18 @@ const ( insightTokenFile = "insight.token" - projectEndpoint = "" projectListEndpoint = "project/list?ids=" timeEntryForUserInRangeEndpoint = "time-entry/user-date-range?modelProperties=ActualHours,Date,Description,Project_Id,Task_Id,TimeSheet_Id,User_Id&startDate=2020-01-01&endDate=%s&userId=%s" userEndpoint = "user/list-active?modelProperties=Id,FirstName,LastName,EmailAddress" ) type ( - Interconnect struct { - Client *IClient - DBConnector *DBConnector - } - - IClient struct { + InsightClient struct { client *http.Client tokens *map[string]string } - Project struct { + InsightProject struct { BillableExpenseVarianceEarnedValueBillableVsWorkBillableTotalPercent float64 `json:"BillableExpense_VarianceEarnedValueBillableVsWorkBillableTotalPercent,omitempty"` CompanyDefaultId string `json:"CompanyDefault_Id,omitempty"` CustomFieldValueId string `json:"CustomFieldValue_Id,omitempty"` @@ -79,7 +72,7 @@ type ( WorkSeconds int64 } - TimeEntry struct { + InsightTimeEntry struct { ActualHours float64 TimeEntryDate string `json:"Date,omitempty"` TimeEntryDescription string `json:"Description,omitempty"` @@ -89,7 +82,7 @@ type ( UserId string `json:"User_Id,omitempty"` } - User struct { + InsightUser struct { Id string FirstName string LastName string @@ -97,57 +90,13 @@ type ( } ) -// -/*====================================================================================== - interconnect -======================================================================================*/ - -func NewInsightConnect() *Interconnect { - connect := Interconnect{} - connect.Client = NewIClient() - connect.DBConnector = &DBConnector{} - return &connect -} - -func (ic *Interconnect) ResetTables() { - ic.DBConnector.CreateTables() -} - -func (ic *Interconnect) UpdateUsers() { - users := ic.Client.GetUsers() - ic.DBConnector.UpdateUsers(users) -} - -func (ic *Interconnect) UpdateTimeEntries() { - users := ic.DBConnector.FetchUsers() - entryChan := make(chan *[]*TimeEntry) - coroutineCount := 0 - dateString := createDateString() - for _, userPtr := range *users { - go ic.Client.GetTimeAllTimeEntriesForUserThroughDate(userPtr.Id, dateString, entryChan) - coroutineCount++ - time.Sleep(1000 * time.Millisecond) - } - - log.Printf("Currently pending goroutines: %d\n", coroutineCount) - for coroutineCount > 0 { - entries := <-entryChan - ic.DBConnector.UpdateTimeEntries(entries) - coroutineCount-- - log.Printf("Currently pending goroutines: %d\n", coroutineCount) - } - ic.DBConnector.ExecuteSqlScript("insight", "create-insight-contribution-table.sql") -} - -// - -// +// /*====================================================================================== - IClient + InsightClient ======================================================================================*/ -func NewIClient() *IClient { - c := IClient{ +func NewIClient() *InsightClient { + c := InsightClient{ client: &http.Client{}, tokens: cacheTokens(), } @@ -155,12 +104,12 @@ func NewIClient() *IClient { return &c } -func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string, returnChan chan<- *[]*TimeEntry) { +func (iClient *InsightClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string, returnChan chan<- *[]*InsightTimeEntry) { urlString := fmt.Sprintf(timeEntryForUserInRangeEndpoint, endDate, userId) log.Println(urlString) req := iClient.createRequest(urlString, insightTokenFile) rawBytes := iClient.doGet(req) - container := make([]*TimeEntry, 0, 1000) //do we want to make the default result size parametric? + container := make([]*InsightTimeEntry, 0, 1000) //do we want to make the default result size parametric? err := json.Unmarshal(*rawBytes, &container) if err != nil { log.Panic(err) @@ -168,11 +117,11 @@ func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, e returnChan <- &container } -func (iClient *IClient) GetProjectsInList(projectIds []string) *[]Project { +func (iClient *InsightClient) GetProjectsInList(projectIds []string) *[]InsightProject { ids := strings.Join(projectIds, ",") req := iClient.createRequest(projectListEndpoint+ids, insightTokenFile) rawBytes := iClient.doGet(req) - container := make([]Project, 0, 1000) //do we want to make the default result size parametric? + container := make([]InsightProject, 0, 1000) //do we want to make the default result size parametric? err := json.Unmarshal(*rawBytes, &container) if err != nil { @@ -182,7 +131,7 @@ func (iClient *IClient) GetProjectsInList(projectIds []string) *[]Project { return &container } -func (iClient *IClient) GetUsers() *[]User { +func (iClient *InsightClient) GetUsers() *[]*InsightUser { users := iClient.getRawUsers() for idx, user := range *users { user.EmailAddress = strings.ToLower(user.EmailAddress) @@ -193,10 +142,10 @@ func (iClient *IClient) GetUsers() *[]User { return users } -func (iClient *IClient) getRawUsers() *[]User { +func (iClient *InsightClient) getRawUsers() *[]*InsightUser { req := iClient.createRequest(userEndpoint, insightTokenFile) rawBytes := iClient.doGet(req) - container := make([]User, 0, 150) //do we want to make the default result size parametric? + container := make([]*InsightUser, 0, 150) //do we want to make the default result size parametric? err := json.Unmarshal(*rawBytes, &container) if err != nil { @@ -206,7 +155,7 @@ func (iClient *IClient) getRawUsers() *[]User { return &container } -func (iClient *IClient) doGet(req *http.Request) *[]byte { +func (iClient *InsightClient) doGet(req *http.Request) *[]byte { resp, err := iClient.client.Do(req) if err != nil { log.Panic(err) @@ -220,7 +169,7 @@ func (iClient *IClient) doGet(req *http.Request) *[]byte { return &bodyBytes } -func (iClient *IClient) createRequest(urlEndpoint string, tokenFile string) *http.Request { +func (iClient *InsightClient) createRequest(urlEndpoint string, tokenFile string) *http.Request { reqUrl := BaseApiUrl + urlEndpoint token := (*iClient.tokens)[tokenFile] request, err := http.NewRequest("GET", reqUrl, nil) @@ -258,9 +207,4 @@ func cacheTokens() *map[string]string { return &ret } -func createDateString() string { - now := time.Now() - return now.Format("2006-01-02") -} - // diff --git a/src/projectInsight/insight-database.go b/src/projectInsight/insight-database.go new file mode 100644 index 0000000..5959004 --- /dev/null +++ b/src/projectInsight/insight-database.go @@ -0,0 +1,107 @@ +package projectInsight + +import ( + "database/sql" + _ "github.com/go-sql-driver/mysql" + "log" + "mercury/src/util" +) + +const ( + InsightDatabaseName = "insight" +) + +var ( + insightUserMappingFunction = func(rows *sql.Rows) *InsightUser { + user := InsightUser{} + err := rows.Scan(&user.Id, &user.FirstName, &user.LastName, &user.EmailAddress) + if err != nil { + log.Panic(err) + } + return &user + } + + insightTimeEntryUpdateFunction = func(s *sql.Stmt, item *InsightTimeEntry) { + _, err := s.Exec(item.ActualHours, + item.TimeEntryDate, + item.TimeEntryDescription, + item.ProjectId, + item.TaskId, + item.TimeSheetId, + item.UserId) + if err != nil { + log.Printf("%#v\n", s) + log.Panic(err) + } + } + + insightUserUpdateFunction = func(s *sql.Stmt, item *InsightUser) { + _, err := s.Exec(item.Id, item.FirstName, item.LastName, item.EmailAddress) + if err != nil { + log.Printf("%#v\n", s) + log.Panic(err) + } + } +) + +// +/*====================================================================================== + InsightDBConnector +======================================================================================*/ + +type InsightDBConnector struct { + *util.DBConnectorGeneric +} + +func NewDBConnection() *InsightDBConnector { + dbGeneric := util.DBConnectorGeneric{} + return &InsightDBConnector{&dbGeneric} +} + +func (c *InsightDBConnector) CreateTables() { + tableCreationScripts := []string{"create-insight-user-table.sql", "create-insight-timeentry-table.sql"} + + for _, scriptName := range tableCreationScripts { + c.ExecuteSqlScript(InsightDatabaseName, scriptName) + } +} + +func (c *InsightDBConnector) UpdateTimeEntries(entries *[]*InsightTimeEntry) { + util.BulkUpdate[InsightTimeEntry]( + c.DBConnectorGeneric, + InsightDatabaseName, + "update-insight-timeEntry.sql", + entries, + insightTimeEntryUpdateFunction, + ) +} + +func (c *InsightDBConnector) UpdateUsers(users *[]*InsightUser) { + util.BulkUpdate[InsightUser]( + c.DBConnectorGeneric, + InsightDatabaseName, + "update-insight-timeEntry.sql", + users, + insightUserUpdateFunction, + ) +} + +func (c *InsightDBConnector) ReadUsers() *[]*InsightUser { + return util.QueryForObjects[InsightUser]( + c.DBConnectorGeneric, + InsightDatabaseName, + "read-insight-allUsers.sql", + insightUserMappingFunction, + ) +} + +func (c *InsightDBConnector) ReadEngineerUsers() *[]*InsightUser { + return util.QueryForObjects[InsightUser]( + c.DBConnectorGeneric, + InsightDatabaseName, + "read-insight-engineerUsers.sql", + insightUserMappingFunction, + ) +} + +// diff --git a/src/sql/create-timeentry-table.sql b/src/sql/create-insight-timeentry-table.sql similarity index 100% rename from src/sql/create-timeentry-table.sql rename to src/sql/create-insight-timeentry-table.sql diff --git a/src/sql/read-clarity-allProjects.sql b/src/sql/read-clarity-allProjects.sql new file mode 100644 index 0000000..d90ebb9 --- /dev/null +++ b/src/sql/read-clarity-allProjects.sql @@ -0,0 +1 @@ +SELECT * FROM projects.all_projects; \ No newline at end of file diff --git a/src/sql/read-insight-allUsers.sql b/src/sql/read-insight-allUsers.sql new file mode 100644 index 0000000..b28910e --- /dev/null +++ b/src/sql/read-insight-allUsers.sql @@ -0,0 +1 @@ +SELECT * FROM users; \ No newline at end of file diff --git a/src/sql/read-insight-engineerUsers.sql b/src/sql/read-insight-engineerUsers.sql new file mode 100644 index 0000000..4bdc902 --- /dev/null +++ b/src/sql/read-insight-engineerUsers.sql @@ -0,0 +1,4 @@ +SELECT insight.users.Id, insight.users.FirstName, insight.users.LastName, insight.users.EmailAddress +FROM insight.users + INNER JOIN projects.users on insight.users.EmailAddress = projects.users.email +where projects.users.priv & POW(2, 25) > 0; \ No newline at end of file diff --git a/src/sql/update-timeentry.sql b/src/sql/update-insight-timeEntry.sql similarity index 100% rename from src/sql/update-timeentry.sql rename to src/sql/update-insight-timeEntry.sql diff --git a/src/sql/update-users.sql b/src/sql/update-insight-users.sql similarity index 100% rename from src/sql/update-users.sql rename to src/sql/update-insight-users.sql diff --git a/src/util/database-primitives.go b/src/util/database-primitives.go new file mode 100644 index 0000000..bfe7c16 --- /dev/null +++ b/src/util/database-primitives.go @@ -0,0 +1,130 @@ +package util + +import ( + "database/sql" + "fmt" + "io/ioutil" + "log" + "os" + "strings" +) + +const ( + dbCredsEnvName = "DB_CREDS" + dbCredsHostName = "DB_HOST" +) + +// +/*====================================================================================== + DBConnectorGeneric +======================================================================================*/ + +type DBConnectorGeneric struct { + cachedConnection *sql.DB +} + +func (c *DBConnectorGeneric) ExecuteSqlScript(database string, scriptName string) { + c.StartConnection(database) + defer c.ReturnConnection() + queryWhole := *loadSqlFile(scriptName) + queryParts := strings.Split(queryWhole, ";") + + log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole) + + for _, query := range queryParts { + if len(query) == 0 || query == "\n" { + continue + } + _, err := c.cachedConnection.Exec(query + ";") + if err != nil { + log.Printf("Query involved in error: %s\n", query) + log.Panic(err) + } + } +} + +func (c *DBConnectorGeneric) StartConnection(dataBase string) { + db := createDbConnection(dataBase) + c.cachedConnection = db +} + +func (c *DBConnectorGeneric) ReturnConnection() { + err := c.cachedConnection.Close() + if err != nil { + log.Panic(err) + } + c.cachedConnection = nil +} + +func (c *DBConnectorGeneric) QueryFromScript(scriptName string) *sql.Rows { + query := loadSqlFile(scriptName) + rows, err := c.cachedConnection.Query(*query) + if err != nil { + log.Panic(err) + } + return rows +} + +// + +// +/*====================================================================================== + Utility Functions +======================================================================================*/ + +func createDbConnection(database string) *sql.DB { + cred := os.Getenv(dbCredsEnvName) + host := os.Getenv(dbCredsHostName) + dbString := "clarity:%s@tcp(%s)/%s" + connectString := fmt.Sprintf(dbString, cred, host, database) + + db, err := sql.Open("mysql", connectString) + if err != nil { + log.Panic(err) + } + return db +} + +func loadSqlFile(scriptName string) *string { + file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755) + if err != nil { + log.Panic(err) + } + defer file.Close() + raw, err := ioutil.ReadAll(file) + if err != nil { + log.Panic(err) + } + str := strings.Trim(string(raw), "\n") + return &str +} + +func QueryForObjects[K any](db *DBConnectorGeneric, dbName string, queryScript string, mapping func(rows *sql.Rows) *K) *[]*K { + rs := make([]*K, 0, 10000) + db.StartConnection(dbName) + defer db.ReturnConnection() + + res := db.QueryFromScript(queryScript) + for res.Next() { + o := mapping(res) + rs = append(rs, o) + } + + return &rs +} + +func BulkUpdate[K any](db *DBConnectorGeneric, dbName string, updateScript string, items *[]*K, mapping func(s *sql.Stmt, item *K)) { + db.StartConnection(dbName) + defer db.ReturnConnection() + statement := loadSqlFile(updateScript) + s, err := db.cachedConnection.Prepare(*statement) + if err != nil { + log.Panic(err) + } + for _, item := range *items { + mapping(s, item) + } + +} + +//