refactoring of the database is complete and a lot more clean. It is currently untested, so we'll kick off the inaugural scan here in a minute.
dtookey
4 years ago
parent
f1a149ee52
commit
e40a72f38e
@ -0,0 +1 @@
|
||||
package clarity
|
||||
@ -0,0 +1,32 @@
|
||||
package clarity
|
||||
|
||||
import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"log"
|
||||
)
|
||||
|
||||
func GetS3ListingForKey(region string, bucket string, key string) *[]*s3.Object {
|
||||
ses, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String(region),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
|
||||
conn := s3.New(ses)
|
||||
|
||||
req := s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(key),
|
||||
}
|
||||
|
||||
resp, err := conn.ListObjectsV2(&req)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
|
||||
return &resp.Contents
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
package clarity
|
||||
|
||||
import (
|
||||
"log"
|
||||
"mercury/src/projectInsight"
|
||||
)
|
||||
|
||||
//<editor-fold name="Snitch">
|
||||
/*======================================================================================
|
||||
Snitch
|
||||
======================================================================================*/
|
||||
|
||||
type Snitch struct {
|
||||
DB projectInsight.InsightDBConnector
|
||||
}
|
||||
|
||||
//func NewSnitch() *Snitch {
|
||||
// return &Snitch{}
|
||||
//}
|
||||
|
||||
func (s *Snitch) Test() {
|
||||
|
||||
res := GetS3ListingForKey("us-east-1", "jds.private.rdu.str", "RDU2201010/")
|
||||
|
||||
for _, thing := range *res {
|
||||
log.Println(*thing.Key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Snitch) getProjectNumbersForTesting() *[]*string {
|
||||
return nil
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold name="utility functions">
|
||||
/*======================================================================================
|
||||
utility functions
|
||||
======================================================================================*/
|
||||
|
||||
//</editor-fold>
|
||||
@ -1,194 +0,0 @@
|
||||
package insight
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
InsightDatabaseName = "insight"
|
||||
dbCredsEnvName = "DB_CREDS"
|
||||
dbCredsHostName = "DB_HOST"
|
||||
)
|
||||
|
||||
//<editor-fold name="DBConnector">
|
||||
/*======================================================================================
|
||||
DBConnector
|
||||
======================================================================================*/
|
||||
|
||||
type DBConnector struct{}
|
||||
|
||||
func NewDBConnection() *DBConnector {
|
||||
return &DBConnector{}
|
||||
}
|
||||
|
||||
func (c *DBConnector) CreateTables() {
|
||||
tableCreationScripts := []string{"create-insight-user-table.sql", "create-timeentry-table.sql"}
|
||||
|
||||
for _, scriptName := range tableCreationScripts {
|
||||
c.ExecuteSqlScript(InsightDatabaseName, scriptName)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DBConnector) UpdateTimeEntries(entries *[]*TimeEntry) {
|
||||
statement := loadSqlFile("update-timeentry.sql")
|
||||
db := c.checkoutConnection(InsightDatabaseName)
|
||||
defer c.returnConnection(db)
|
||||
s, err := db.Prepare(*statement)
|
||||
defer s.Close()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
for _, ent := range *entries {
|
||||
_, err = s.Exec(ent.ActualHours,
|
||||
ent.TimeEntryDate,
|
||||
ent.TimeEntryDescription,
|
||||
ent.ProjectId,
|
||||
ent.TaskId,
|
||||
ent.TimeSheetId,
|
||||
ent.UserId)
|
||||
if err != nil {
|
||||
log.Printf("%#v\n", s)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DBConnector) UpdateUsers(users *[]User) {
|
||||
statement := loadSqlFile("update-users.sql")
|
||||
db := c.checkoutConnection(InsightDatabaseName)
|
||||
defer c.returnConnection(db)
|
||||
s, err := db.Prepare(*statement)
|
||||
defer s.Close()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
for _, user := range *users {
|
||||
_, err = s.Exec(user.Id, user.FirstName, user.LastName, user.EmailAddress)
|
||||
if err != nil {
|
||||
log.Printf("%#v\n", s)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DBConnector) FetchUsers() *[]*User {
|
||||
ret := make([]*User, 0, 50)
|
||||
cx := c.checkoutConnection(InsightDatabaseName)
|
||||
queryText := "SELECT * FROM users;"
|
||||
rs, err := cx.Query(queryText)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
for rs.Next() {
|
||||
u := User{}
|
||||
err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
ret = append(ret, &u)
|
||||
}
|
||||
return &ret
|
||||
}
|
||||
|
||||
func (c *DBConnector) FetchEngineerUsers() *[]*User {
|
||||
ret := make([]*User, 0, 50)
|
||||
cx := c.checkoutConnection(InsightDatabaseName)
|
||||
queryText := "SELECT insight.users.Id, insight.users.FirstName, insight.users.LastName, insight.users.EmailAddress FROM insight.users INNER JOIN projects.users on insight.users.EmailAddress = projects.users.email where projects.users.priv & POW(2, 25) > 0;"
|
||||
rs, err := cx.Query(queryText)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
for rs.Next() {
|
||||
u := User{}
|
||||
err := rs.Scan(&u.Id, &u.FirstName, &u.LastName, &u.EmailAddress)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
ret = append(ret, &u)
|
||||
}
|
||||
return &ret
|
||||
}
|
||||
|
||||
func (c *DBConnector) ExecuteSqlScript(database string, scriptName string) {
|
||||
db := c.checkoutConnection(database)
|
||||
defer c.returnConnection(db)
|
||||
queryWhole := *loadSqlFile(scriptName)
|
||||
queryParts := strings.Split(queryWhole, ";")
|
||||
|
||||
log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole)
|
||||
|
||||
for _, query := range queryParts {
|
||||
if len(query) == 0 || query == "\n" {
|
||||
continue
|
||||
}
|
||||
_, err := db.Exec(query + ";")
|
||||
if err != nil {
|
||||
log.Printf("Query involved in error: %s\n", query)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DBConnector) checkoutConnection(dataBase string) *sql.DB {
|
||||
return createDbConnection(dataBase)
|
||||
}
|
||||
|
||||
func (c *DBConnector) returnConnection(db *sql.DB) {
|
||||
db.Close()
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold name="Utility Functions">
|
||||
/*======================================================================================
|
||||
Utility Functions
|
||||
======================================================================================*/
|
||||
|
||||
func createDbConnection(database string) *sql.DB {
|
||||
cred := os.Getenv(dbCredsEnvName)
|
||||
host := os.Getenv(dbCredsHostName)
|
||||
dbString := "clarity:%s@tcp(%s)/%s"
|
||||
connectString := fmt.Sprintf(dbString, cred, host, database)
|
||||
|
||||
db, err := sql.Open("mysql", connectString)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func getSecret(pathlike string) *string {
|
||||
file, err := os.OpenFile(pathlike, os.O_RDONLY, 0755)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
raw, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
str := strings.Trim(string(raw), "\n")
|
||||
return &str
|
||||
}
|
||||
|
||||
func loadSqlFile(scriptName string) *string {
|
||||
file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
raw, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
str := strings.Trim(string(raw), "\n")
|
||||
return &str
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
@ -0,0 +1,68 @@
|
||||
package mercury
|
||||
|
||||
import (
|
||||
"log"
|
||||
"mercury/src/projectInsight"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Interconnect struct {
|
||||
Client *projectInsight.InsightClient
|
||||
InsightDBConnector *projectInsight.InsightDBConnector
|
||||
}
|
||||
|
||||
//<editor-fold name="interconnect">
|
||||
/*======================================================================================
|
||||
interconnect
|
||||
======================================================================================*/
|
||||
|
||||
func NewInterconnect() *Interconnect {
|
||||
connect := Interconnect{}
|
||||
connect.Client = projectInsight.NewIClient()
|
||||
connect.InsightDBConnector = &projectInsight.InsightDBConnector{}
|
||||
return &connect
|
||||
}
|
||||
|
||||
func (ic *Interconnect) ResetTables() {
|
||||
ic.InsightDBConnector.CreateTables()
|
||||
}
|
||||
|
||||
func (ic *Interconnect) UpdateUsers() {
|
||||
users := ic.Client.GetUsers()
|
||||
ic.InsightDBConnector.UpdateUsers(users)
|
||||
}
|
||||
|
||||
func (ic *Interconnect) UpdateTimeEntries() {
|
||||
users := ic.InsightDBConnector.ReadUsers()
|
||||
entryChan := make(chan *[]*projectInsight.InsightTimeEntry)
|
||||
coroutineCount := 0
|
||||
dateString := createDateString()
|
||||
for _, userPtr := range *users {
|
||||
go ic.Client.GetTimeAllTimeEntriesForUserThroughDate(userPtr.Id, dateString, entryChan)
|
||||
coroutineCount++
|
||||
time.Sleep(1000 * time.Millisecond)
|
||||
}
|
||||
|
||||
log.Printf("Currently pending goroutines: %d\n", coroutineCount)
|
||||
for coroutineCount > 0 {
|
||||
entries := <-entryChan
|
||||
ic.InsightDBConnector.UpdateTimeEntries(entries)
|
||||
coroutineCount--
|
||||
log.Printf("Currently pending goroutines: %d\n", coroutineCount)
|
||||
}
|
||||
ic.InsightDBConnector.ExecuteSqlScript("insight", "create-insight-contribution-table.sql")
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold name="UtilityFunctions">
|
||||
/*======================================================================================
|
||||
UtilityFunctions
|
||||
======================================================================================*/
|
||||
|
||||
func createDateString() string {
|
||||
now := time.Now()
|
||||
return now.Format("2006-01-02")
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
@ -0,0 +1,107 @@
|
||||
package projectInsight
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"log"
|
||||
"mercury/src/util"
|
||||
)
|
||||
|
||||
const (
|
||||
InsightDatabaseName = "insight"
|
||||
)
|
||||
|
||||
var (
|
||||
insightUserMappingFunction = func(rows *sql.Rows) *InsightUser {
|
||||
user := InsightUser{}
|
||||
err := rows.Scan(&user.Id, &user.FirstName, &user.LastName, &user.EmailAddress)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
return &user
|
||||
}
|
||||
|
||||
insightTimeEntryUpdateFunction = func(s *sql.Stmt, item *InsightTimeEntry) {
|
||||
_, err := s.Exec(item.ActualHours,
|
||||
item.TimeEntryDate,
|
||||
item.TimeEntryDescription,
|
||||
item.ProjectId,
|
||||
item.TaskId,
|
||||
item.TimeSheetId,
|
||||
item.UserId)
|
||||
if err != nil {
|
||||
log.Printf("%#v\n", s)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
insightUserUpdateFunction = func(s *sql.Stmt, item *InsightUser) {
|
||||
_, err := s.Exec(item.Id, item.FirstName, item.LastName, item.EmailAddress)
|
||||
if err != nil {
|
||||
log.Printf("%#v\n", s)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
//<editor-fold name="InsightDBConnector">
|
||||
/*======================================================================================
|
||||
InsightDBConnector
|
||||
======================================================================================*/
|
||||
|
||||
type InsightDBConnector struct {
|
||||
*util.DBConnectorGeneric
|
||||
}
|
||||
|
||||
func NewDBConnection() *InsightDBConnector {
|
||||
dbGeneric := util.DBConnectorGeneric{}
|
||||
return &InsightDBConnector{&dbGeneric}
|
||||
}
|
||||
|
||||
func (c *InsightDBConnector) CreateTables() {
|
||||
tableCreationScripts := []string{"create-insight-user-table.sql", "create-insight-timeentry-table.sql"}
|
||||
|
||||
for _, scriptName := range tableCreationScripts {
|
||||
c.ExecuteSqlScript(InsightDatabaseName, scriptName)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *InsightDBConnector) UpdateTimeEntries(entries *[]*InsightTimeEntry) {
|
||||
util.BulkUpdate[InsightTimeEntry](
|
||||
c.DBConnectorGeneric,
|
||||
InsightDatabaseName,
|
||||
"update-insight-timeEntry.sql",
|
||||
entries,
|
||||
insightTimeEntryUpdateFunction,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *InsightDBConnector) UpdateUsers(users *[]*InsightUser) {
|
||||
util.BulkUpdate[InsightUser](
|
||||
c.DBConnectorGeneric,
|
||||
InsightDatabaseName,
|
||||
"update-insight-timeEntry.sql",
|
||||
users,
|
||||
insightUserUpdateFunction,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *InsightDBConnector) ReadUsers() *[]*InsightUser {
|
||||
return util.QueryForObjects[InsightUser](
|
||||
c.DBConnectorGeneric,
|
||||
InsightDatabaseName,
|
||||
"read-insight-allUsers.sql",
|
||||
insightUserMappingFunction,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *InsightDBConnector) ReadEngineerUsers() *[]*InsightUser {
|
||||
return util.QueryForObjects[InsightUser](
|
||||
c.DBConnectorGeneric,
|
||||
InsightDatabaseName,
|
||||
"read-insight-engineerUsers.sql",
|
||||
insightUserMappingFunction,
|
||||
)
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
@ -0,0 +1 @@
|
||||
SELECT * FROM projects.all_projects;
|
||||
@ -0,0 +1 @@
|
||||
SELECT * FROM users;
|
||||
@ -0,0 +1,4 @@
|
||||
SELECT insight.users.Id, insight.users.FirstName, insight.users.LastName, insight.users.EmailAddress
|
||||
FROM insight.users
|
||||
INNER JOIN projects.users on insight.users.EmailAddress = projects.users.email
|
||||
where projects.users.priv & POW(2, 25) > 0;
|
||||
@ -0,0 +1,130 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
dbCredsEnvName = "DB_CREDS"
|
||||
dbCredsHostName = "DB_HOST"
|
||||
)
|
||||
|
||||
//<editor-fold name="DBConnectorGeneric">
|
||||
/*======================================================================================
|
||||
DBConnectorGeneric
|
||||
======================================================================================*/
|
||||
|
||||
type DBConnectorGeneric struct {
|
||||
cachedConnection *sql.DB
|
||||
}
|
||||
|
||||
func (c *DBConnectorGeneric) ExecuteSqlScript(database string, scriptName string) {
|
||||
c.StartConnection(database)
|
||||
defer c.ReturnConnection()
|
||||
queryWhole := *loadSqlFile(scriptName)
|
||||
queryParts := strings.Split(queryWhole, ";")
|
||||
|
||||
log.Printf("\n=============================================\n%s\n==============================================\n", queryWhole)
|
||||
|
||||
for _, query := range queryParts {
|
||||
if len(query) == 0 || query == "\n" {
|
||||
continue
|
||||
}
|
||||
_, err := c.cachedConnection.Exec(query + ";")
|
||||
if err != nil {
|
||||
log.Printf("Query involved in error: %s\n", query)
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *DBConnectorGeneric) StartConnection(dataBase string) {
|
||||
db := createDbConnection(dataBase)
|
||||
c.cachedConnection = db
|
||||
}
|
||||
|
||||
func (c *DBConnectorGeneric) ReturnConnection() {
|
||||
err := c.cachedConnection.Close()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
c.cachedConnection = nil
|
||||
}
|
||||
|
||||
func (c *DBConnectorGeneric) QueryFromScript(scriptName string) *sql.Rows {
|
||||
query := loadSqlFile(scriptName)
|
||||
rows, err := c.cachedConnection.Query(*query)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold name="Utility Functions">
|
||||
/*======================================================================================
|
||||
Utility Functions
|
||||
======================================================================================*/
|
||||
|
||||
func createDbConnection(database string) *sql.DB {
|
||||
cred := os.Getenv(dbCredsEnvName)
|
||||
host := os.Getenv(dbCredsHostName)
|
||||
dbString := "clarity:%s@tcp(%s)/%s"
|
||||
connectString := fmt.Sprintf(dbString, cred, host, database)
|
||||
|
||||
db, err := sql.Open("mysql", connectString)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func loadSqlFile(scriptName string) *string {
|
||||
file, err := os.OpenFile("src/sql/"+scriptName, os.O_RDONLY, 0755)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
raw, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
str := strings.Trim(string(raw), "\n")
|
||||
return &str
|
||||
}
|
||||
|
||||
func QueryForObjects[K any](db *DBConnectorGeneric, dbName string, queryScript string, mapping func(rows *sql.Rows) *K) *[]*K {
|
||||
rs := make([]*K, 0, 10000)
|
||||
db.StartConnection(dbName)
|
||||
defer db.ReturnConnection()
|
||||
|
||||
res := db.QueryFromScript(queryScript)
|
||||
for res.Next() {
|
||||
o := mapping(res)
|
||||
rs = append(rs, o)
|
||||
}
|
||||
|
||||
return &rs
|
||||
}
|
||||
|
||||
func BulkUpdate[K any](db *DBConnectorGeneric, dbName string, updateScript string, items *[]*K, mapping func(s *sql.Stmt, item *K)) {
|
||||
db.StartConnection(dbName)
|
||||
defer db.ReturnConnection()
|
||||
statement := loadSqlFile(updateScript)
|
||||
s, err := db.cachedConnection.Prepare(*statement)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
for _, item := range *items {
|
||||
mapping(s, item)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//</editor-fold>
|
||||
Loading…
Reference in New Issue