From 8e105637e3a802f303807d801b779788923f933d Mon Sep 17 00:00:00 2001 From: dtookey Date: Tue, 26 Apr 2022 12:39:54 -0400 Subject: [PATCH] started work on transferring information to the data-connect database --- go.mod | 4 + src/insight/insight-connect.go | 165 +++++++++++++++++++++++++---- src/insight/insight-database.go | 148 ++++++++++++++++++++++++++ src/mercury.go | 32 +++++- src/sql/create-timeentry-table.sql | 21 ++++ src/sql/create-user-table.sql | 8 ++ src/sql/update-timeentry.sql | 17 +++ src/sql/update-users.sql | 1 + 8 files changed, 372 insertions(+), 24 deletions(-) create mode 100644 src/insight/insight-database.go create mode 100644 src/sql/create-timeentry-table.sql create mode 100644 src/sql/create-user-table.sql create mode 100644 src/sql/update-timeentry.sql create mode 100644 src/sql/update-users.sql diff --git a/go.mod b/go.mod index ddcd8c9..b836343 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,7 @@ module mercury go 1.18 + +require ( + github.com/go-sql-driver/mysql v1.6.0 +) \ No newline at end of file diff --git a/src/insight/insight-connect.go b/src/insight/insight-connect.go index 62effbd..3bcda25 100644 --- a/src/insight/insight-connect.go +++ b/src/insight/insight-connect.go @@ -7,14 +7,72 @@ import ( "log" "net/http" "os" - "strconv" + "strings" ) const ( BaseApiUrl = "https://jds.projectinsight.net/api/" + + insightTokenFile = "insight.token" + + projectEndpoint = "" + projectListEndpoint = "project/list?ids=" + timeEntryForUserInRangeEndpoint = "time-entry/user-date-range?startDate=2020-01-01&endDate=%s&userId=%s" + userEndpoint = "user/list-active?modelProperties=Id,FirstName,LastName,EmailAddress" ) type ( + IClient struct { + client *http.Client + tokens *map[string]string + } + + Project struct { + BillableExpenseVarianceEarnedValueBillableVsWorkBillableTotalPercent float64 `json:"BillableExpense_VarianceEarnedValueBillableVsWorkBillableTotalPercent,omitempty"` + CompanyDefaultId string `json:"CompanyDefault_Id,omitempty"` + CustomFieldValueId string `json:"CustomFieldValue_Id,omitempty"` + DepartmentId string `json:"Department_Id,omitempty"` + DurationSeconds int64 + DurationString string + EndDateTimeUTC string + EstimateAtCompletionToTargetTotalVariance float64 `json:"EstimateAtCompletionToTargetTotal_Variance,omitempty"` + EstimateAtCompletionToTargetTotalPercentVariance float64 `json:"EstimateAtCompletionToTargetTotalPercent_Variance,omitempty"` + IsActive bool + IsOpen bool + LocationDefaultId string `json:"LocationDefault_Id,omitempty"` + Name string + ProfitSalesOrderActual float64 + ProjectPriorityId string `json:"ProjectPriority_Id,omitempty"` + ProjectState int64 `json:"project_state,omitempty"` + ProjectStatusId string `json:"ProjectStatus_Id,omitempty"` + ProjectTypeId string `json:"ProjectType_Id,omitempty"` + ScheduleStartDate string + StartDateTimeUTC string + UrlFull string + VarSalesOrderExpenseVsBillableExpense float64 + VarSalesOrderExpenseVsBillableExpensePercent float64 + VarSalesOrderExpenseVsInvoiceExpense float64 + VarSalesOrderExpenseVsInvoiceExpensePercent float64 + VarSalesOrderHoursVsBillableHours float64 + VarSalesOrderHoursVsBillableHoursPercent float64 + VarSalesOrderHoursVsInvoiceHours float64 + VarSalesOrderHoursVsInvoiceHoursPercent float64 + VarSalesOrderRateVsBillableRate float64 + VarSalesOrderRateVsBillableRatePercent float64 + VarSalesOrderRateVsInvoiceRate float64 + VarSalesOrderRateVsInvoiceRatePercent float64 + VarSalesOrderTimeVsBillableTime float64 + VarSalesOrderTimeVsBillableTimePercent float64 + VarSalesOrderTimeVsInvoiceTime float64 + VarSalesOrderTimeVsInvoiceTimePercent float64 + VarSalesOrderTotalVsBillableTotal float64 + VarSalesOrderTotalVsBillableTotalPercent float64 + VarSalesOrderTotalVsInvoiceTotal float64 + VarSalesOrderTotalVsInvoiceTotalPercent float64 + WorkPercentComplete float64 + WorkSeconds int64 + } + TimeEntry struct { ActualHours float64 ActualHoursFormattedString string @@ -33,54 +91,117 @@ type ( TimeSheetId string `json:"TimeSheet_Id,omitempty"` UserId string `json:"User_Id,omitempty"` } -) -func Get(urlEndpoint string) { - client := http.Client{} - reqUrl := BaseApiUrl + urlEndpoint - token := getToken("insight.token") - deserializeContainer := make([]TimeEntry, 0, 20000) + User struct { + Id string + FirstName string + LastName string + EmailAddress string + } +) - request, err := http.NewRequest("GET", reqUrl, nil) +func (iClient *IClient) GetTimeAllTimeEntriesForUserThroughDate(userId string, endDate string) *[]TimeEntry { + urlString := fmt.Sprintf(timeEntryForUserInRangeEndpoint, endDate, userId) + log.Println(urlString) + req := iClient.createRequest(urlString, insightTokenFile) + rawBytes := iClient.doGet(req) + container := make([]TimeEntry, 0, 1000) //do we want to make the default result size parametric? + log.Println(string(*rawBytes)) + err := json.Unmarshal(*rawBytes, &container) if err != nil { log.Fatal(err) } + return &container +} - request.Header.Add("api-token", token) +func (iClient *IClient) GetProjectsInList(projectIds []string) *[]Project { + ids := strings.Join(projectIds, ",") + req := iClient.createRequest(projectListEndpoint+ids, insightTokenFile) + rawBytes := iClient.doGet(req) + container := make([]Project, 0, 1000) //do we want to make the default result size parametric? - resp, err := client.Do(request) + err := json.Unmarshal(*rawBytes, &container) if err != nil { log.Fatal(err) } - bodyBytes, err := ioutil.ReadAll(resp.Body) + return &container +} + +func (iClient *IClient) GetUsers() *[]User { + users := iClient.getRawUsers() + for idx, user := range *users { + user.EmailAddress = strings.ToLower(user.EmailAddress) + user.EmailAddress = strings.ReplaceAll(user.EmailAddress, "jdsdesignonline.com", "jdsconsulting.net") + user.EmailAddress = strings.ReplaceAll(user.EmailAddress, "jdsfaulkner.com", "jdsconsulting.net") + (*users)[idx] = user + } + return users +} + +func (iClient *IClient) getRawUsers() *[]User { + req := iClient.createRequest(userEndpoint, insightTokenFile) + rawBytes := iClient.doGet(req) + container := make([]User, 0, 150) //do we want to make the default result size parametric? + + err := json.Unmarshal(*rawBytes, &container) if err != nil { log.Fatal(err) } - err = json.Unmarshal(bodyBytes, &deserializeContainer) + return &container +} + +func NewIClient() *IClient { + c := IClient{ + client: &http.Client{}, + tokens: cacheTokens(), + } + + return &c +} + +func cacheTokens() *map[string]string { + files, err := os.ReadDir("./tokens/") + ret := make(map[string]string) if err != nil { - log.Fatal(err) + panic(err) } + for _, file := range files { + subPath := fmt.Sprintf("./tokens/%s", file.Name()) + fileContents, err := os.ReadFile(subPath) + if err != nil { + log.Fatal(err) + } - for _, entry := range deserializeContainer { - log.Printf("%#v\n", entry) + ret[file.Name()] = string(fileContents) } - log.Println("Received Entries (qty): " + strconv.Itoa(len(deserializeContainer))) + return &ret } -func getToken(tokenName string) string { - filePath := fmt.Sprintf("./tokens/%s", tokenName) - file, err := os.OpenFile(filePath, os.O_RDONLY, 0755) +func (iClient *IClient) doGet(req *http.Request) *[]byte { + resp, err := iClient.client.Do(req) if err != nil { log.Fatal(err) } - defer file.Close() - rawBytes, err := ioutil.ReadAll(file) + bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { log.Fatal(err) } - return string(rawBytes) + return &bodyBytes +} + +func (iClient *IClient) createRequest(urlEndpoint string, tokenFile string) *http.Request { + reqUrl := BaseApiUrl + urlEndpoint + token := (*iClient.tokens)[tokenFile] + request, err := http.NewRequest("GET", reqUrl, nil) + if err != nil { + log.Fatal(err) + } + + request.Header.Add("api-token", token) + + return request } diff --git a/src/insight/insight-database.go b/src/insight/insight-database.go new file mode 100644 index 0000000..41517ca --- /dev/null +++ b/src/insight/insight-database.go @@ -0,0 +1,148 @@ +package insight + +import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" + "io/ioutil" + "log" + "os" + "strings" +) + +const ( + InsightDatabaseName = "insight" +) + +type Connector struct{} + +func NewInsightConnection() *Connector { + return &Connector{} +} + +func (c *Connector) CreateTables() { + tableCreationScripts := []string{"create-user-table.sql", "create-timeentry-table.sql"} + + for _, scriptName := range tableCreationScripts { + c.ExecuteSqlScript(InsightDatabaseName, scriptName) + } +} + +func (c *Connector) UpdateTimeEntries(entries *[]TimeEntry) { + statement := loadSqlFile("update-timeentry.sql") + db := c.checkoutConnection(InsightDatabaseName) + defer c.returnConnection(db) + s, err := db.Prepare(*statement) + defer s.Close() + if err != nil { + log.Fatalln(err) + } + for _, ent := range *entries { + _, err = s.Exec(ent.ActualHours, + ent.ActualHoursFormattedString, + ent.ActualTimeString, + ent.ActualTotal, + ent.BillableHours, + ent.BillableHoursFormattedString, + ent.BillableTimeString, + ent.BillableTotal, + ent.Date, + ent.Description, + ent.ProjectId, + ent.RateBill, + ent.RateBurden, + ent.TaskId, + ent.TimeSheetId, + ent.UserId) + if err != nil { + log.Printf("%#v\n", s) + log.Fatalln(err) + } + } +} + +func (c *Connector) UpdateUsers(users *[]User) { + statement := loadSqlFile("update-users.sql") + db := c.checkoutConnection(InsightDatabaseName) + defer c.returnConnection(db) + s, err := db.Prepare(*statement) + defer s.Close() + if err != nil { + log.Fatalln(err) + } + for _, user := range *users { + _, err = s.Exec(user.Id, user.FirstName, user.LastName, user.EmailAddress) + if err != nil { + log.Printf("%#v\n", s) + log.Fatalln(err) + } + } +} + +func (c *Connector) ExecuteSqlScript(database string, scriptName string) { + db := c.checkoutConnection(database) + defer c.returnConnection(db) + queryWhole := *loadSqlFile(scriptName) + queryParts := strings.Split(queryWhole, ";") + + log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole) + + for _, query := range queryParts { + if len(query) == 0 || query == "\n" { + continue + } + _, err := db.Exec(query + ";") + if err != nil { + log.Printf("Query involved in error: %s\n", query) + log.Fatal(err) + } + } +} + +func (c *Connector) checkoutConnection(dataBase string) *sql.DB { + return createDbConnection(dataBase) +} + +func (c *Connector) returnConnection(db *sql.DB) { + db.Close() +} + +func createDbConnection(database string) *sql.DB { + secret := getSecret("/home/dtookey/work/datastudio-db-creds.txt") + dbString := "clarity:%s@tcp(data-connect.carolina.engineering)/%s" + connectString := fmt.Sprintf(dbString, *secret, database) + + db, err := sql.Open("mysql", connectString) + if err != nil { + log.Fatalln(err) + } + return db +} + +func getSecret(pathlike string) *string { + file, err := os.OpenFile(pathlike, os.O_RDONLY, 0755) + if err != nil { + log.Fatalln(err) + } + defer file.Close() + raw, err := ioutil.ReadAll(file) + if err != nil { + log.Fatalln(err) + } + str := strings.Trim(string(raw), "\n") + return &str +} + +func loadSqlFile(scriptName string) *string { + file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755) + if err != nil { + log.Fatalln(err) + } + defer file.Close() + raw, err := ioutil.ReadAll(file) + if err != nil { + log.Fatalln(err) + } + str := strings.Trim(string(raw), "\n") + return &str +} diff --git a/src/mercury.go b/src/mercury.go index 4b653de..fec9698 100644 --- a/src/mercury.go +++ b/src/mercury.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "mercury/src/insight" "mercury/src/mercury" @@ -8,7 +9,23 @@ import ( ) func main() { - fetchInsightData() + //fetchInsightData() + test() +} + +func test() { + insightClient := insight.NewIClient() + db := insight.NewInsightConnection() + log.Println("Creating tables") + db.CreateTables() + log.Println("fetching users") + users := insightClient.GetUsers() + log.Println("updating users") + db.UpdateUsers(users) + //log.Println("fetching Time Entries") + //timeEntries := insightClient.GetTimeAllTimeEntriesForUserThroughDate() + //log.Println("updating Time Entries") + //db.UpdateUsers(users) } func processQbBilling() { @@ -23,5 +40,16 @@ func processQbBilling() { func fetchInsightData() { //jcrouch id = e779e0a9-0e56-4dbe-98dd-e8d0048d109f //id := "e779e0a9-0e56-4dbe-98dd-e8d0048d109f" - insight.Get("time-entry/search?startDate=2018-01-01&endDate=2022-04-01") + client := insight.NewIClient() + users := client.GetUsers() + for _, user := range *users { + + if user.EmailAddress == "jcrouch@jdsconsulting.net" { + fmt.Println(user.EmailAddress) + timeEntries := client.GetTimeAllTimeEntriesForUserThroughDate(user.Id, "2022-04-22") + for _, entry := range *timeEntries { + fmt.Printf("%#v\n", entry) + } + } + } } diff --git a/src/sql/create-timeentry-table.sql b/src/sql/create-timeentry-table.sql new file mode 100644 index 0000000..42dd560 --- /dev/null +++ b/src/sql/create-timeentry-table.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS insight.timeentry; + +CREATE TABLE insight.timeentry +( + ActualHours REAL, + ActualHoursFormattedString varchar(30), + ActualTimeString VARCHAR(30), + ActualTotal REAL, + BillableHours REAL, + BillableHoursFormattedString VARCHAR(30), + BillableTimeString VARCHAR(30), + BillableTotal REAL, + Date VARCHAR(30), + Description VARCHAR(255), + ProjectId VARCHAR(50), + RateBill REAL, + RateBurden REAL, + TaskId VARCHAR(50), + TimeSheetId VARCHAR(50), + UserId VARCHAR(50) +); \ No newline at end of file diff --git a/src/sql/create-user-table.sql b/src/sql/create-user-table.sql new file mode 100644 index 0000000..133f101 --- /dev/null +++ b/src/sql/create-user-table.sql @@ -0,0 +1,8 @@ +DROP TABLE IF EXISTS insight.users; + +CREATE TABLE insight.users ( + Id varchar(75) UNIQUE PRIMARY KEY, + FirstName VARCHAR(75), + LastName VARCHAR(75), + EmailAddress VARCHAR(150) +); \ No newline at end of file diff --git a/src/sql/update-timeentry.sql b/src/sql/update-timeentry.sql new file mode 100644 index 0000000..2ac7d2f --- /dev/null +++ b/src/sql/update-timeentry.sql @@ -0,0 +1,17 @@ +INSERT INTO insight.timeentry (ActualHours, + ActualHoursFormattedString, + ActualTimeString, + ActualTotal, + BillableHours, + BillableHoursFormattedString, + BillableTimeString, + BillableTotal, + Date, + Description, + ProjectId, + RateBill, + RateBurden, + TaskId, + TimeSheetId, + UserId) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); \ No newline at end of file diff --git a/src/sql/update-users.sql b/src/sql/update-users.sql new file mode 100644 index 0000000..e9850d5 --- /dev/null +++ b/src/sql/update-users.sql @@ -0,0 +1 @@ +INSERT INTO insight.users (Id, FirstName, LastName, EmailAddress) VALUES (?, ?, ?, ?); \ No newline at end of file