created abstractions for automatically pulling users and time entries for users. We haven't completed a full successful (the description field in TimeEntry failed at varchar(512), so we've upped it to 4k) run. However, it's currently running and has gotten past the initial hiccup.

src/insight/insight-connect.go

@@ -22,6 +22,11 @@ const (
 )
 
 type (
+	Interconnect struct {
+		Client      *IClient
+		DBConnector *DBConnector
+	}
+
 	IClient struct {
 		client *http.Client
 		tokens *map[string]string
@@ -100,13 +105,58 @@ type (
 	}
 )
 
-func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]TimeEntry {
+//<editor-fold name="interconnect">
+/*======================================================================================
+                              interconnect
+======================================================================================*/
+
+func NewInsightConnect() *Interconnect {
+	connect := Interconnect{}
+	connect.Client = NewIClient()
+	connect.DBConnector = &DBConnector{}
+	return &connect
+}
+
+func (ic *Interconnect) ResetTables() {
+	ic.DBConnector.CreateTables()
+}
+
+func (ic *Interconnect) UpdateUsers() {
+	users := ic.Client.GetUsers()
+	ic.DBConnector.UpdateUsers(users)
+}
+
+func (ic *Interconnect) UpdateTimeEntries() {
+	users := ic.DBConnector.FetchUsers()
+	for _, userPtr := range *users {
+		user := *userPtr
+		entries := ic.Client.GetTimeAllTimeEntriesForUserThroughDate(user.Id, "2022-04-28")
+		ic.DBConnector.UpdateTimeEntries(entries)
+	}
+}
+
+//</editor-fold>
+
+//<editor-fold name="IClient">
+/*======================================================================================
+                              IClient
+======================================================================================*/
+
+func NewIClient() *IClient {
+	c := IClient{
+		client: &http.Client{},
+		tokens: cacheTokens(),
+	}
+
+	return &c
+}
+
+func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]*TimeEntry {
 	urlString := fmt.Sprintf(timeEntryForUserInRangeEndpoint, endDate, userId)
 	log.Println(urlString)
 	req := iClient.createRequest(urlString, insightTokenFile)
 	rawBytes := iClient.doGet(req)
-	container := make([]TimeEntry, 0, 1000) //do we want to make the default result size parametric?
+	container := make([]*TimeEntry, 0, 1000) //do we want to make the default result size parametric?
-	log.Println(string(*rawBytes))
 	err := json.Unmarshal(*rawBytes, &container)
 	if err != nil {
 		log.Fatal(err)
@@ -152,33 +202,6 @@ func (iClient *IClient) getRawUsers() *[]User {
 	return &container
 }
 
-func NewIClient() *IClient {
-	c := IClient{
-		client: &http.Client{},
-		tokens: cacheTokens(),
-	}
-
-	return &c
-}
-
-func cacheTokens() *map[string]string {
-	files, err := os.ReadDir("./tokens/")
-	ret := make(map[string]string)
-	if err != nil {
-		panic(err)
-	}
-	for _, file := range files {
-		subPath := fmt.Sprintf("./tokens/%s", file.Name())
-		fileContents, err := os.ReadFile(subPath)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		ret[file.Name()] = string(fileContents)
-	}
-	return &ret
-}
-
 func (iClient *IClient) doGet(req *http.Request) *[]byte {
 	resp, err := iClient.client.Do(req)
 	if err != nil {
@@ -205,3 +228,30 @@ func (iClient *IClient) createRequest(urlEndpoint string, tokenFile string) *htt
 
 	return request
 }
+
+//</editor-fold>
+
+//<editor-fold name="utility functions">
+/*======================================================================================
+                              utility functions
+======================================================================================*/
+
+func cacheTokens() *map[string]string {
+	files, err := os.ReadDir("./tokens/")
+	ret := make(map[string]string)
+	if err != nil {
+		panic(err)
+	}
+	for _, file := range files {
+		subPath := fmt.Sprintf("./tokens/%s", file.Name())
+		fileContents, err := os.ReadFile(subPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		ret[file.Name()] = string(fileContents)
+	}
+	return &ret
+}
+
+//</editor-fold>

src/insight/insight-database.go

@@ -14,13 +14,18 @@ const (
 	InsightDatabaseName = "insight"
 )
 
-type Connector struct{}
+//<editor-fold name="DBConnector">
+/*======================================================================================
+                              DBConnector
+======================================================================================*/
 
-func NewInsightConnection() *Connector {
+type DBConnector struct{}
-	return &Connector{}
+
+func NewDBConnection() *DBConnector {
+	return &DBConnector{}
 }
 
-func (c *Connector) CreateTables() {
+func (c *DBConnector) CreateTables() {
 	tableCreationScripts := []string{"create-user-table.sql", "create-timeentry-table.sql"}
 
 	for _, scriptName := range tableCreationScripts {
@@ -28,7 +33,7 @@ func (c *Connector) CreateTables() {
 	}
 }
 
-func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) {
+func (c *DBConnector) UpdateTimeEntries(entries *[]*TimeEntry) {
 	statement := loadSqlFile("update-timeentry.sql")
 	db := c.checkoutConnection(InsightDatabaseName)
 	defer c.returnConnection(db)
@@ -61,7 +66,7 @@ func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) {
 	}
 }
 
-func (c *Connector) UpdateUsers(users *[]User) {
+func (c *DBConnector) UpdateUsers(users *[]User) {
 	statement := loadSqlFile("update-users.sql")
 	db := c.checkoutConnection(InsightDatabaseName)
 	defer c.returnConnection(db)
@@ -79,7 +84,26 @@ func (c *Connector) UpdateUsers(users *[]User) {
 	}
 }
 
-func (c *Connector) ExecuteSqlScript(database string, scriptName string) {
+func (c *DBConnector) FetchUsers() *[]*User {
+	ret := make([]*User, 0, 50)
+	cx := c.checkoutConnection(InsightDatabaseName)
+	queryText := "SELECT * FROM users;"
+	rs, err := cx.Query(queryText)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for rs.Next() {
+		u := User{}
+		err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		ret = append(ret, &u)
+	}
+	return &ret
+}
+
+func (c *DBConnector) ExecuteSqlScript(database string, scriptName string) {
 	db := c.checkoutConnection(database)
 	defer c.returnConnection(db)
 	queryWhole := *loadSqlFile(scriptName)
@@ -99,14 +123,21 @@ func (c *Connector) ExecuteSqlScript(database string, scriptName string) {
 	}
 }
 
-func (c *Connector) checkoutConnection(dataBase string) *sql.DB {
+func (c *DBConnector) checkoutConnection(dataBase string) *sql.DB {
 	return createDbConnection(dataBase)
 }
 
-func (c *Connector) returnConnection(db *sql.DB) {
+func (c *DBConnector) returnConnection(db *sql.DB) {
 	db.Close()
 }
 
+//</editor-fold>
+
+//<editor-fold name="Utility Functions">
+/*======================================================================================
+                              Utility Functions
+======================================================================================*/
+
 func createDbConnection(database string) *sql.DB {
 	secret := getSecret("/home/dtookey/work/datastudio-db-creds.txt")
 	dbString := "clarity:%s@tcp(data-connect.carolina.engineering)/%s"
@@ -146,3 +177,5 @@ func loadSqlFile(scriptName string) *string {
 	str := strings.Trim(string(raw), "\n")
 	return &str
 }
+
+//</editor-fold>

src/mercury.go

@@ -12,26 +12,19 @@ import (
 func main() {
 	s := time.Now()
 
-	processQbBilling()
+	//processQbBilling()
+	test()
+	//fetchInsightData()
 	f := time.Now()
 	log.Println(f.Sub(s).Microseconds())
-	//fetchInsightData()
+
-	//test()
 }
 
 func test() {
-	insightClient := insight.NewIClient()
+	icx := insight.NewInsightConnect()
-	db := insight.NewInsightConnection()
+	icx.ResetTables()
-	log.Println("Creating tables")
+	icx.UpdateUsers()
-	db.CreateTables()
+	icx.UpdateTimeEntries()
-	log.Println("fetching users")
-	users := insightClient.GetUsers()
-	log.Println("updating users")
-	db.UpdateUsers(users)
-	//log.Println("fetching Time Entries")
-	//timeEntries := insightClient.GetTimeAllTimeEntriesForUserThroughDate()
-	//log.Println("updating Time Entries")
-	//db.UpdateUsers(users)
 }
 
 func processQbBilling() {

src/sql/create-timeentry-table.sql

@@ -11,7 +11,7 @@ CREATE TABLE insight.timeentry
     BillableTimeString           VARCHAR(30),
     BillableTotal                REAL,
     Date                         VARCHAR(30),
-    Description                  VARCHAR(255),
+    Description                  VARCHAR(4096),
     ProjectId                    VARCHAR(50),
     RateBill                     REAL,
     RateBurden                   REAL,