started work on transferring information to the data-connect database

go.mod

@@ -1,3 +1,7 @@
 module mercury
 
 go 1.18
+
+require (
+	github.com/go-sql-driver/mysql v1.6.0
+)

src/insight/insight-connect.go

@@ -7,14 +7,72 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"strconv"
+	"strings"
 )
 
 const (
 	BaseApiUrl = "https://jds.projectinsight.net/api/"
+
+	insightTokenFile = "insight.token"
+
+	projectEndpoint                 = ""
+	projectListEndpoint             = "project/list?ids="
+	timeEntryForUserInRangeEndpoint = "time-entry/user-date-range?startDate=2020-01-01&endDate=%s&userId=%s"
+	userEndpoint                    = "user/list-active?modelProperties=Id,FirstName,LastName,EmailAddress"
 )
 
 type (
+	IClient struct {
+		client *http.Client
+		tokens *map[string]string
+	}
+
+	Project struct {
+		BillableExpenseVarianceEarnedValueBillableVsWorkBillableTotalPercent float64 `json:"BillableExpense_VarianceEarnedValueBillableVsWorkBillableTotalPercent,omitempty"`
+		CompanyDefaultId                                                     string  `json:"CompanyDefault_Id,omitempty"`
+		CustomFieldValueId                                                   string  `json:"CustomFieldValue_Id,omitempty"`
+		DepartmentId                                                         string  `json:"Department_Id,omitempty"`
+		DurationSeconds                                                      int64
+		DurationString                                                       string
+		EndDateTimeUTC                                                       string
+		EstimateAtCompletionToTargetTotalVariance                            float64 `json:"EstimateAtCompletionToTargetTotal_Variance,omitempty"`
+		EstimateAtCompletionToTargetTotalPercentVariance                     float64 `json:"EstimateAtCompletionToTargetTotalPercent_Variance,omitempty"`
+		IsActive                                                             bool
+		IsOpen                                                               bool
+		LocationDefaultId                                                    string `json:"LocationDefault_Id,omitempty"`
+		Name                                                                 string
+		ProfitSalesOrderActual                                               float64
+		ProjectPriorityId                                                    string `json:"ProjectPriority_Id,omitempty"`
+		ProjectState                                                         int64  `json:"project_state,omitempty"`
+		ProjectStatusId                                                      string `json:"ProjectStatus_Id,omitempty"`
+		ProjectTypeId                                                        string `json:"ProjectType_Id,omitempty"`
+		ScheduleStartDate                                                    string
+		StartDateTimeUTC                                                     string
+		UrlFull                                                              string
+		VarSalesOrderExpenseVsBillableExpense                                float64
+		VarSalesOrderExpenseVsBillableExpensePercent                         float64
+		VarSalesOrderExpenseVsInvoiceExpense                                 float64
+		VarSalesOrderExpenseVsInvoiceExpensePercent                          float64
+		VarSalesOrderHoursVsBillableHours                                    float64
+		VarSalesOrderHoursVsBillableHoursPercent                             float64
+		VarSalesOrderHoursVsInvoiceHours                                     float64
+		VarSalesOrderHoursVsInvoiceHoursPercent                              float64
+		VarSalesOrderRateVsBillableRate                                      float64
+		VarSalesOrderRateVsBillableRatePercent                               float64
+		VarSalesOrderRateVsInvoiceRate                                       float64
+		VarSalesOrderRateVsInvoiceRatePercent                                float64
+		VarSalesOrderTimeVsBillableTime                                      float64
+		VarSalesOrderTimeVsBillableTimePercent                               float64
+		VarSalesOrderTimeVsInvoiceTime                                       float64
+		VarSalesOrderTimeVsInvoiceTimePercent                                float64
+		VarSalesOrderTotalVsBillableTotal                                    float64
+		VarSalesOrderTotalVsBillableTotalPercent                             float64
+		VarSalesOrderTotalVsInvoiceTotal                                     float64
+		VarSalesOrderTotalVsInvoiceTotalPercent                              float64
+		WorkPercentComplete                                                  float64
+		WorkSeconds                                                          int64
+	}
+
 	TimeEntry struct {
 		ActualHours                  float64
 		ActualHoursFormattedString   string
@@ -33,54 +91,117 @@ type (
 		TimeSheetId                  string `json:"TimeSheet_Id,omitempty"`
 		UserId                       string `json:"User_Id,omitempty"`
 	}
-)
 
-func Get(urlEndpoint string) {
-	client := http.Client{}
-	reqUrl := BaseApiUrl + urlEndpoint
-	token := getToken("insight.token")
-	deserializeContainer := make([]TimeEntry, 0, 20000)
+	User struct {
+		Id           string
+		FirstName    string
+		LastName     string
+		EmailAddress string
+	}
+)
 
-	request, err := http.NewRequest("GET", reqUrl, nil)
+func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]TimeEntry {
+	urlString := fmt.Sprintf(timeEntryForUserInRangeEndpoint, endDate, userId)
+	log.Println(urlString)
+	req := iClient.createRequest(urlString, insightTokenFile)
+	rawBytes := iClient.doGet(req)
+	container := make([]TimeEntry, 0, 1000) //do we want to make the default result size parametric?
+	log.Println(string(*rawBytes))
+	err := json.Unmarshal(*rawBytes, &container)
 	if err != nil {
 		log.Fatal(err)
 	}
+	return &container
+}
 
-	request.Header.Add("api-token", token)
+func (iClient *IClient) GetProjectsInList(projectIds []string) *[]Project {
+	ids := strings.Join(projectIds, ",")
+	req := iClient.createRequest(projectListEndpoint+ids, insightTokenFile)
+	rawBytes := iClient.doGet(req)
+	container := make([]Project, 0, 1000) //do we want to make the default result size parametric?
 
-	resp, err := client.Do(request)
+	err := json.Unmarshal(*rawBytes, &container)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	return &container
+}
+
+func (iClient *IClient) GetUsers() *[]User {
+	users := iClient.getRawUsers()
+	for idx, user := range *users {
+		user.EmailAddress = strings.ToLower(user.EmailAddress)
+		user.EmailAddress = strings.ReplaceAll(user.EmailAddress, "jdsdesignonline.com", "jdsconsulting.net")
+		user.EmailAddress = strings.ReplaceAll(user.EmailAddress, "jdsfaulkner.com", "jdsconsulting.net")
+		(*users)[idx] = user
+	}
+	return users
+}
+
+func (iClient *IClient) getRawUsers() *[]User {
+	req := iClient.createRequest(userEndpoint, insightTokenFile)
+	rawBytes := iClient.doGet(req)
+	container := make([]User, 0, 150) //do we want to make the default result size parametric?
+
+	err := json.Unmarshal(*rawBytes, &container)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	err = json.Unmarshal(bodyBytes, &deserializeContainer)
+	return &container
+}
+
+func NewIClient() *IClient {
+	c := IClient{
+		client: &http.Client{},
+		tokens: cacheTokens(),
+	}
+
+	return &c
+}
+
+func cacheTokens() *map[string]string {
+	files, err := os.ReadDir("./tokens/")
+	ret := make(map[string]string)
 	if err != nil {
-		log.Fatal(err)
+		panic(err)
 	}
+	for _, file := range files {
+		subPath := fmt.Sprintf("./tokens/%s", file.Name())
+		fileContents, err := os.ReadFile(subPath)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	for _, entry := range deserializeContainer {
-		log.Printf("%#v\n", entry)
+		ret[file.Name()] = string(fileContents)
 	}
-	log.Println("Received Entries (qty): " + strconv.Itoa(len(deserializeContainer)))
+	return &ret
 }
 
-func getToken(tokenName string) string {
-	filePath := fmt.Sprintf("./tokens/%s", tokenName)
-	file, err := os.OpenFile(filePath, os.O_RDONLY, 0755)
+func (iClient *IClient) doGet(req *http.Request) *[]byte {
+	resp, err := iClient.client.Do(req)
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer file.Close()
 
-	rawBytes, err := ioutil.ReadAll(file)
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	return string(rawBytes)
+	return &bodyBytes
+}
+
+func (iClient *IClient) createRequest(urlEndpoint string, tokenFile string) *http.Request {
+	reqUrl := BaseApiUrl + urlEndpoint
+	token := (*iClient.tokens)[tokenFile]
+	request, err := http.NewRequest("GET", reqUrl, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	request.Header.Add("api-token", token)
+
+	return request
 }

src/insight/insight-database.go

@@ -0,0 +1,148 @@
+package insight
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/go-sql-driver/mysql"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+const (
+	InsightDatabaseName = "insight"
+)
+
+type Connector struct{}
+
+func NewInsightConnection() *Connector {
+	return &Connector{}
+}
+
+func (c *Connector) CreateTables() {
+	tableCreationScripts := []string{"create-user-table.sql", "create-timeentry-table.sql"}
+
+	for _, scriptName := range tableCreationScripts {
+		c.ExecuteSqlScript(InsightDatabaseName, scriptName)
+	}
+}
+
+func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) {
+	statement := loadSqlFile("update-timeentry.sql")
+	db := c.checkoutConnection(InsightDatabaseName)
+	defer c.returnConnection(db)
+	s, err := db.Prepare(*statement)
+	defer s.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for _, ent := range *entries {
+		_, err = s.Exec(ent.ActualHours,
+			ent.ActualHoursFormattedString,
+			ent.ActualTimeString,
+			ent.ActualTotal,
+			ent.BillableHours,
+			ent.BillableHoursFormattedString,
+			ent.BillableTimeString,
+			ent.BillableTotal,
+			ent.Date,
+			ent.Description,
+			ent.ProjectId,
+			ent.RateBill,
+			ent.RateBurden,
+			ent.TaskId,
+			ent.TimeSheetId,
+			ent.UserId)
+		if err != nil {
+			log.Printf("%#v\n", s)
+			log.Fatalln(err)
+		}
+	}
+}
+
+func (c *Connector) UpdateUsers(users *[]User) {
+	statement := loadSqlFile("update-users.sql")
+	db := c.checkoutConnection(InsightDatabaseName)
+	defer c.returnConnection(db)
+	s, err := db.Prepare(*statement)
+	defer s.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for _, user := range *users {
+		_, err = s.Exec(user.Id, user.FirstName, user.LastName, user.EmailAddress)
+		if err != nil {
+			log.Printf("%#v\n", s)
+			log.Fatalln(err)
+		}
+	}
+}
+
+func (c *Connector) ExecuteSqlScript(database string, scriptName string) {
+	db := c.checkoutConnection(database)
+	defer c.returnConnection(db)
+	queryWhole := *loadSqlFile(scriptName)
+	queryParts := strings.Split(queryWhole, ";")
+
+	log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole)
+
+	for _, query := range queryParts {
+		if len(query) == 0 || query == "\n" {
+			continue
+		}
+		_, err := db.Exec(query + ";")
+		if err != nil {
+			log.Printf("Query involved in error: %s\n", query)
+			log.Fatal(err)
+		}
+	}
+}
+
+func (c *Connector) checkoutConnection(dataBase string) *sql.DB {
+	return createDbConnection(dataBase)
+}
+
+func (c *Connector) returnConnection(db *sql.DB) {
+	db.Close()
+}
+
+func createDbConnection(database string) *sql.DB {
+	secret := getSecret("/home/dtookey/work/datastudio-db-creds.txt")
+	dbString := "clarity:%s@tcp(data-connect.carolina.engineering)/%s"
+	connectString := fmt.Sprintf(dbString, *secret, database)
+
+	db, err := sql.Open("mysql", connectString)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	return db
+}
+
+func getSecret(pathlike string) *string {
+	file, err := os.OpenFile(pathlike, os.O_RDONLY, 0755)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer file.Close()
+	raw, err := ioutil.ReadAll(file)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	str := strings.Trim(string(raw), "\n")
+	return &str
+}
+
+func loadSqlFile(scriptName string) *string {
+	file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer file.Close()
+	raw, err := ioutil.ReadAll(file)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	str := strings.Trim(string(raw), "\n")
+	return &str
+}

src/mercury.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"mercury/src/insight"
 	"mercury/src/mercury"
@@ -8,7 +9,23 @@ import (
 )
 
 func main() {
-	fetchInsightData()
+	//fetchInsightData()
+	test()
+}
+
+func test() {
+	insightClient := insight.NewIClient()
+	db := insight.NewInsightConnection()
+	log.Println("Creating tables")
+	db.CreateTables()
+	log.Println("fetching users")
+	users := insightClient.GetUsers()
+	log.Println("updating users")
+	db.UpdateUsers(users)
+	//log.Println("fetching Time Entries")
+	//timeEntries := insightClient.GetTimeAllTimeEntriesForUserThroughDate()
+	//log.Println("updating Time Entries")
+	//db.UpdateUsers(users)
 }
 
 func processQbBilling() {
@@ -23,5 +40,16 @@ func processQbBilling() {
 func fetchInsightData() {
 	//jcrouch id = e779e0a9-0e56-4dbe-98dd-e8d0048d109f
 	//id := "e779e0a9-0e56-4dbe-98dd-e8d0048d109f"
-	insight.Get("time-entry/search?startDate=2018-01-01&endDate=2022-04-01")
+	client := insight.NewIClient()
+	users := client.GetUsers()
+	for _, user := range *users {
+
+		if user.EmailAddress == "jcrouch@jdsconsulting.net" {
+			fmt.Println(user.EmailAddress)
+			timeEntries := client.GetTimeAllTimeEntriesForUserThroughDate(user.Id, "2022-04-22")
+			for _, entry := range *timeEntries {
+				fmt.Printf("%#v\n", entry)
+			}
+		}
+	}
 }

src/sql/create-timeentry-table.sql

@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS insight.timeentry;
+
+CREATE TABLE insight.timeentry
+(
+    ActualHours                  REAL,
+    ActualHoursFormattedString   varchar(30),
+    ActualTimeString             VARCHAR(30),
+    ActualTotal                  REAL,
+    BillableHours                REAL,
+    BillableHoursFormattedString VARCHAR(30),
+    BillableTimeString           VARCHAR(30),
+    BillableTotal                REAL,
+    Date                         VARCHAR(30),
+    Description                  VARCHAR(255),
+    ProjectId                    VARCHAR(50),
+    RateBill                     REAL,
+    RateBurden                   REAL,
+    TaskId                       VARCHAR(50),
+    TimeSheetId                  VARCHAR(50),
+    UserId                       VARCHAR(50)
+);

src/sql/create-user-table.sql

@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS insight.users;
+
+CREATE TABLE insight.users (
+    Id varchar(75) UNIQUE PRIMARY KEY,
+    FirstName VARCHAR(75),
+    LastName VARCHAR(75),
+    EmailAddress VARCHAR(150)
+);

src/sql/update-timeentry.sql

@@ -0,0 +1,17 @@
+INSERT INTO insight.timeentry (ActualHours,
+                               ActualHoursFormattedString,
+                               ActualTimeString,
+                               ActualTotal,
+                               BillableHours,
+                               BillableHoursFormattedString,
+                               BillableTimeString,
+                               BillableTotal,
+                               Date,
+                               Description,
+                               ProjectId,
+                               RateBill,
+                               RateBurden,
+                               TaskId,
+                               TimeSheetId,
+                               UserId)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);

src/sql/update-users.sql

@@ -0,0 +1 @@
+INSERT INTO insight.users (Id, FirstName, LastName, EmailAddress) VALUES (?, ?, ?, ?);