From 49e6b7bb7300c8eb3fb4b43a86562f51e8fc3e28 Mon Sep 17 00:00:00 2001 From: dtookey Date: Thu, 28 Apr 2022 16:49:04 -0400 Subject: [PATCH] created abstractions for automatically pulling users and time entries for users. We haven't completed a full successful (the description field in TimeEntry failed at varchar(512), so we've upped it to 4k) run. However, it's currently running and has gotten past the initial hiccup. --- src/insight/insight-connect.go | 110 +++++++++++++++++++++-------- src/insight/insight-database.go | 51 ++++++++++--- src/mercury.go | 23 +++--- src/sql/create-timeentry-table.sql | 2 +- 4 files changed, 131 insertions(+), 55 deletions(-) diff --git a/src/insight/insight-connect.go b/src/insight/insight-connect.go index 3bcda25..b531711 100644 --- a/src/insight/insight-connect.go +++ b/src/insight/insight-connect.go @@ -22,6 +22,11 @@ const ( ) type ( + Interconnect struct { + Client *IClient + DBConnector *DBConnector + } + IClient struct { client *http.Client tokens *map[string]string @@ -100,13 +105,58 @@ type ( } ) -func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]TimeEntry { +// +/*====================================================================================== + interconnect +======================================================================================*/ + +func NewInsightConnect() *Interconnect { + connect := Interconnect{} + connect.Client = NewIClient() + connect.DBConnector = &DBConnector{} + return &connect +} + +func (ic *Interconnect) ResetTables() { + ic.DBConnector.CreateTables() +} + +func (ic *Interconnect) UpdateUsers() { + users := ic.Client.GetUsers() + ic.DBConnector.UpdateUsers(users) +} + +func (ic *Interconnect) UpdateTimeEntries() { + users := ic.DBConnector.FetchUsers() + for _, userPtr := range *users { + user := *userPtr + entries := ic.Client.GetTimeAllTimeEntriesForUserThroughDate(user.Id, "2022-04-28") + ic.DBConnector.UpdateTimeEntries(entries) + } +} + +// + +// +/*====================================================================================== + IClient +======================================================================================*/ + +func NewIClient() *IClient { + c := IClient{ + client: &http.Client{}, + tokens: cacheTokens(), + } + + return &c +} + +func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]*TimeEntry { urlString := fmt.Sprintf(timeEntryForUserInRangeEndpoint, endDate, userId) log.Println(urlString) req := iClient.createRequest(urlString, insightTokenFile) rawBytes := iClient.doGet(req) - container := make([]TimeEntry, 0, 1000) //do we want to make the default result size parametric? - log.Println(string(*rawBytes)) + container := make([]*TimeEntry, 0, 1000) //do we want to make the default result size parametric? err := json.Unmarshal(*rawBytes, &container) if err != nil { log.Fatal(err) @@ -152,33 +202,6 @@ func (iClient *IClient) getRawUsers() *[]User { return &container } -func NewIClient() *IClient { - c := IClient{ - client: &http.Client{}, - tokens: cacheTokens(), - } - - return &c -} - -func cacheTokens() *map[string]string { - files, err := os.ReadDir("./tokens/") - ret := make(map[string]string) - if err != nil { - panic(err) - } - for _, file := range files { - subPath := fmt.Sprintf("./tokens/%s", file.Name()) - fileContents, err := os.ReadFile(subPath) - if err != nil { - log.Fatal(err) - } - - ret[file.Name()] = string(fileContents) - } - return &ret -} - func (iClient *IClient) doGet(req *http.Request) *[]byte { resp, err := iClient.client.Do(req) if err != nil { @@ -205,3 +228,30 @@ func (iClient *IClient) createRequest(urlEndpoint string, tokenFile string) *htt return request } + +// + +// +/*====================================================================================== + utility functions +======================================================================================*/ + +func cacheTokens() *map[string]string { + files, err := os.ReadDir("./tokens/") + ret := make(map[string]string) + if err != nil { + panic(err) + } + for _, file := range files { + subPath := fmt.Sprintf("./tokens/%s", file.Name()) + fileContents, err := os.ReadFile(subPath) + if err != nil { + log.Fatal(err) + } + + ret[file.Name()] = string(fileContents) + } + return &ret +} + +// diff --git a/src/insight/insight-database.go b/src/insight/insight-database.go index 41517ca..2b365ab 100644 --- a/src/insight/insight-database.go +++ b/src/insight/insight-database.go @@ -14,13 +14,18 @@ const ( InsightDatabaseName = "insight" ) -type Connector struct{} +// +/*====================================================================================== + DBConnector +======================================================================================*/ -func NewInsightConnection() *Connector { - return &Connector{} +type DBConnector struct{} + +func NewDBConnection() *DBConnector { + return &DBConnector{} } -func (c *Connector) CreateTables() { +func (c *DBConnector) CreateTables() { tableCreationScripts := []string{"create-user-table.sql", "create-timeentry-table.sql"} for _, scriptName := range tableCreationScripts { @@ -28,7 +33,7 @@ func (c *Connector) CreateTables() { } } -func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) { +func (c *DBConnector) UpdateTimeEntries(entries *[]*TimeEntry) { statement := loadSqlFile("update-timeentry.sql") db := c.checkoutConnection(InsightDatabaseName) defer c.returnConnection(db) @@ -61,7 +66,7 @@ func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) { } } -func (c *Connector) UpdateUsers(users *[]User) { +func (c *DBConnector) UpdateUsers(users *[]User) { statement := loadSqlFile("update-users.sql") db := c.checkoutConnection(InsightDatabaseName) defer c.returnConnection(db) @@ -79,7 +84,26 @@ func (c *Connector) UpdateUsers(users *[]User) { } } -func (c *Connector) ExecuteSqlScript(database string, scriptName string) { +func (c *DBConnector) FetchUsers() *[]*User { + ret := make([]*User, 0, 50) + cx := c.checkoutConnection(InsightDatabaseName) + queryText := "SELECT * FROM users;" + rs, err := cx.Query(queryText) + if err != nil { + log.Fatalln(err) + } + for rs.Next() { + u := User{} + err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress) + if err != nil { + log.Fatalln(err) + } + ret = append(ret, &u) + } + return &ret +} + +func (c *DBConnector) ExecuteSqlScript(database string, scriptName string) { db := c.checkoutConnection(database) defer c.returnConnection(db) queryWhole := *loadSqlFile(scriptName) @@ -99,14 +123,21 @@ func (c *Connector) ExecuteSqlScript(database string, scriptName string) { } } -func (c *Connector) checkoutConnection(dataBase string) *sql.DB { +func (c *DBConnector) checkoutConnection(dataBase string) *sql.DB { return createDbConnection(dataBase) } -func (c *Connector) returnConnection(db *sql.DB) { +func (c *DBConnector) returnConnection(db *sql.DB) { db.Close() } +// + +// +/*====================================================================================== + Utility Functions +======================================================================================*/ + func createDbConnection(database string) *sql.DB { secret := getSecret("/home/dtookey/work/datastudio-db-creds.txt") dbString := "clarity:%s@tcp(data-connect.carolina.engineering)/%s" @@ -146,3 +177,5 @@ func loadSqlFile(scriptName string) *string { str := strings.Trim(string(raw), "\n") return &str } + +// diff --git a/src/mercury.go b/src/mercury.go index 3bc0692..5f4c9a5 100644 --- a/src/mercury.go +++ b/src/mercury.go @@ -12,26 +12,19 @@ import ( func main() { s := time.Now() - processQbBilling() + //processQbBilling() + test() + //fetchInsightData() f := time.Now() log.Println(f.Sub(s).Microseconds()) - //fetchInsightData() - //test() + } func test() { - insightClient := insight.NewIClient() - db := insight.NewInsightConnection() - log.Println("Creating tables") - db.CreateTables() - log.Println("fetching users") - users := insightClient.GetUsers() - log.Println("updating users") - db.UpdateUsers(users) - //log.Println("fetching Time Entries") - //timeEntries := insightClient.GetTimeAllTimeEntriesForUserThroughDate() - //log.Println("updating Time Entries") - //db.UpdateUsers(users) + icx := insight.NewInsightConnect() + icx.ResetTables() + icx.UpdateUsers() + icx.UpdateTimeEntries() } func processQbBilling() { diff --git a/src/sql/create-timeentry-table.sql b/src/sql/create-timeentry-table.sql index 42dd560..f56b84a 100644 --- a/src/sql/create-timeentry-table.sql +++ b/src/sql/create-timeentry-table.sql @@ -11,7 +11,7 @@ CREATE TABLE insight.timeentry BillableTimeString VARCHAR(30), BillableTotal REAL, Date VARCHAR(30), - Description VARCHAR(255), + Description VARCHAR(4096), ProjectId VARCHAR(50), RateBill REAL, RateBurden REAL,