diff --git a/src/db/database-primitives.go b/src/db/database-primitives.go index cc23944..5c82dc0 100644 --- a/src/db/database-primitives.go +++ b/src/db/database-primitives.go @@ -25,6 +25,10 @@ const ( ConnectorGeneric ======================================================================================*/ +type Blocker interface { + ToQueryBlock() string +} + type ConnectorGeneric struct { cachedConnection *sql.DB } @@ -54,6 +58,21 @@ func (c *ConnectorGeneric) ExecuteSqlScript(runner *sqlScriptRunner) { } } +func (c *ConnectorGeneric) ExecuteString(dbName string, payload string) { + c.startConnection(dbName) + defer c.returnConnection() + + //log.Println("==================================================================================================") + //log.Println(payload) + //log.Println("==================================================================================================") + + _, err := c.cachedConnection.Exec(payload) + if err != nil { + log.Printf("Query involved in error: %s\n", payload) + log.Panic(err) + } +} + // startConnection this initializes, caches, and returns a connection for the desired database. If a connection already exists, // it will be terminated and a new connection will be opened. func (c *ConnectorGeneric) startConnection(dataBase string) { @@ -104,6 +123,7 @@ func (c *ConnectorGeneric) CreateTables() { NewRunner("create-insight-timeEntry-table.sql", InsightDatabaseName), NewRunner("create-mercury-picturePatterns-table.sql", MercuryDatabaseName), NewRunner("create-mercury-arReports-table.sql", MercuryDatabaseName), + NewRunner("create-mercury-telecomVoice-table.sql", MercuryDatabaseName), } for _, runner := range tableCreationRunners { @@ -172,6 +192,20 @@ func BulkUpdate[K any](connector *ConnectorGeneric, dbName string, updateScript } } +func BlockUpdate[K Blocker](connector *ConnectorGeneric, dbName string, updateScript string, items *[]*K) { + template := loadSqlFile(updateScript) + buff := strings.Builder{} + l := len(*items) + for i, item := range *items { + buff.WriteString((*item).ToQueryBlock()) + if i < l-1 { + buff.WriteByte(',') + } + } + query := fmt.Sprintf(*template, buff.String()) + connector.ExecuteString(dbName, query) +} + func NewRunner(scriptName string, databaseName string) *sqlScriptRunner { return &sqlScriptRunner{scriptName, databaseName} } diff --git a/src/mercury.go b/src/mercury.go index 6cd21c3..0d02a9a 100644 --- a/src/mercury.go +++ b/src/mercury.go @@ -1,11 +1,10 @@ package main import ( - "fmt" "log" + "mercury/src/db" "mercury/src/finance" "mercury/src/mercury" - "mercury/src/telecom" "os" "time" ) @@ -24,17 +23,10 @@ func main() { } func test() { - - csv := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/call-log-6-7-2022.csv") - results := telecom.ProcessVerizonCsvToObjects(csv, telecom.RowToVerizonCallLine, func(row []string) bool { - if row[5] == "Voice" { - return true - } else { - return false - } - - }) - fmt.Println(len(*results)) + icx := mercury.NewInterconnect() + icx.PseudoInit() + icx.InsightDBConnector.ExecuteSqlScript(db.NewRunner("create-mercury-telecomVoice-table.sql", db.MercuryDatabaseName)) + icx.UpdateVerizonReports() } func updateInsightData() { diff --git a/src/mercury/Interconnect.go b/src/mercury/Interconnect.go index f6e8130..9cdbb44 100644 --- a/src/mercury/Interconnect.go +++ b/src/mercury/Interconnect.go @@ -4,6 +4,7 @@ import ( "log" "mercury/src/db" "mercury/src/projectInsight" + "mercury/src/telecom" "time" ) @@ -23,6 +24,11 @@ func NewInterconnect() *Interconnect { return &connect } +// PseudoInit This is a stupid idea in order to fix the fact that we obliterate tables left-and righta t processing time +func (ic *Interconnect) PseudoInit() { + ic.InsightDBConnector = projectInsight.NewDBConnection() +} + func (ic *Interconnect) Init() { ic.InsightDBConnector = projectInsight.NewDBConnection() ic.InsightDBConnector.ConnectorGeneric.ProcessClarityScripts() @@ -59,6 +65,19 @@ func (ic *Interconnect) UpdateTimeEntries() { ic.InsightDBConnector.ExecuteSqlScript(db.NewRunner("create-insight-contribution-table.sql", "insight")) } +func (ic *Interconnect) UpdateVerizonReports() { + log.Println("Reading CSV Verizon Reports") + callCSV := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/verizon-call.csv") + smsCSV := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/verizon-sms.csv") + callObjects := telecom.ProcessVerizonCsvToObjects(callCSV, telecom.RowToVerizonCallLine, telecom.VerizonCallDataFilter) + smsObjects := telecom.ProcessVerizonCsvToObjects(smsCSV, telecom.RowToVerizonSMSLine, telecom.VerizonSMSDataFilter) + log.Println("Beginning voice data transfer.") + db.BlockUpdate[telecom.VerizonCallLine](ic.InsightDBConnector.ConnectorGeneric, db.MercuryDatabaseName, "update-mercury-voiceCallLine.sql", callObjects) + log.Println("Completed...\nBeginning sms data transfer.") + db.BlockUpdate[telecom.VerizonSMSLine](ic.InsightDBConnector.ConnectorGeneric, db.MercuryDatabaseName, "update-mercury-smsLine.sql", smsObjects) + log.Println("Completed...") +} + // // diff --git a/src/sql/create-mercury-telecomVoice-table.sql b/src/sql/create-mercury-telecomVoice-table.sql new file mode 100644 index 0000000..f53897f --- /dev/null +++ b/src/sql/create-mercury-telecomVoice-table.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS telecom_voice; +DROP TABLE IF EXISTS telecom_sms; + +CREATE TABLE mercury.telecom_voice +( + wireless_number VARCHAR(12), + call_date date, + call_minutes int, + other_number varchar(12) +); + +CREATE TABLE mercury.telecom_sms +( + wireless_number VARCHAR(12), + msg_datetime datetime, + other_number VARCHAR(12), + count_row int default 1 +); + + + + + diff --git a/src/sql/update-mercury-smsLine.sql b/src/sql/update-mercury-smsLine.sql new file mode 100644 index 0000000..89e839c --- /dev/null +++ b/src/sql/update-mercury-smsLine.sql @@ -0,0 +1,3 @@ + +INSERT INTO mercury.telecom_sms (wireless_number, msg_datetime, other_number) +VALUES %s; \ No newline at end of file diff --git a/src/sql/update-mercury-voiceCallLine.sql b/src/sql/update-mercury-voiceCallLine.sql new file mode 100644 index 0000000..2034a2b --- /dev/null +++ b/src/sql/update-mercury-voiceCallLine.sql @@ -0,0 +1,3 @@ + +INSERT INTO mercury.telecom_voice (wireless_number, call_date, call_minutes, other_number) +VALUES %s; \ No newline at end of file diff --git a/src/telecom/voice.go b/src/telecom/verizon.go similarity index 50% rename from src/telecom/voice.go rename to src/telecom/verizon.go index 51f102f..e07362f 100644 --- a/src/telecom/voice.go +++ b/src/telecom/verizon.go @@ -2,22 +2,32 @@ package telecom import ( "bytes" + "database/sql" "encoding/csv" + "fmt" "io/ioutil" + "log" "os" "strconv" + "strings" + "time" ) /* First step is building the reports on verizon's website. Below are the columns (and hopefully categories) that you need to completely recreate the report. - +VerizonCallLine (report structure): Wireless number [mandatory] -"User name" [Contact Information] Date [Voice Usage] Minutes [voice usage] Number [voice usage] +Usage Type [mandatory] + +VerizonSMSLine (report structure): +Wireless number [mandatory] +Message date/time [Messaging usage] +to/from wireless number [Messaging usage] The report comes as a comma/LF formatted CSV. It has 13 lines of useless header data and 1 line of useless total data @@ -25,6 +35,36 @@ that must be trimmed off to shove everything through a struct factory. */ +var VerizonCallLineDBMappingFunction = func(s *sql.Stmt, item *VerizonCallLine) { + _, err := s.Exec( + item.WirelessNumber, + item.CallDate, + item.CallMinutes, + item.OtherNumber, + ) + if err != nil { + log.Printf("%#v\n", s) + log.Panic(err) + } +} + +var VerizonSmsLineDBMappingFunction = func(s *sql.Stmt, item *VerizonSMSLine) { + _, err := s.Exec( + item.WirelessNumber, + item.MsgDateTime, + item.OtherNumber, + ) + if err != nil { + log.Printf("%#v\n", s) + log.Panic(err) + } +} + +const ( + verizonCallHeaderLineCount = 14 + verizonSMSHeaderLineCount = 14 +) + type ( VerizonCSV struct { trimmed bool @@ -34,23 +74,62 @@ type ( VerizonCallLine struct { WirelessNumber string - UserName string CallDate string CallMinutes int OtherNumber string - UsageCategory string + } + + VerizonSMSLine struct { + WirelessNumber string + MsgDateTime string + OtherNumber string } ) func RowToVerizonCallLine(row []string) *VerizonCallLine { - minutes, err := strconv.Atoi(row[3]) + minutes, err := strconv.Atoi(row[2]) if err != nil { + fmt.Printf("%#v\n", row) panic(err) } - line := VerizonCallLine{row[0], row[1], row[2], minutes, row[4], row[5]} + line := VerizonCallLine{row[0], convertDateField(row[1]), minutes, convertSolidPhoneNumberToDashes(row[3])} return &line } +func RowToVerizonSMSLine(row []string) *VerizonSMSLine { + line := VerizonSMSLine{row[0], convertDateTimeField(row[1]), row[2]} + return &line +} + +func VerizonCallDataFilter(row []string) bool { + return len(row[2]) > 0 +} + +func VerizonSMSDataFilter(row []string) bool { + return len(row[2]) > 0 +} + +func convertDateField(d string) string { + parts := strings.Split(d, "/") + return fmt.Sprintf("%s-%s-%s", parts[2], parts[0], parts[1]) +} + +func convertDateTimeField(dt string) string { + dateTime, err := time.Parse("1/2/06 3:04 PM", dt) + if err != nil { + panic(err) + } + + return dateTime.Add(-4 * time.Hour).Format("2006-01-02 15:04:05.000000") +} + +func convertSolidPhoneNumberToDashes(str string) string { + if len(str) < 10 { + return "" + } + return fmt.Sprintf("%s-%s-%s", str[0:3], str[3:6], str[6:10]) +} + func NewVerizonCSV(pathlike string) *VerizonCSV { csv := VerizonCSV{trimmed: false} file, err := os.OpenFile(pathlike, os.O_RDONLY, 777) @@ -81,17 +160,22 @@ func (v *VerizonCSV) trimHeadersAndFooters() { for i, c := range data { if c == '\n' { lineCount += 1 - if lineCount == 13 { + if lineCount == verizonCallHeaderLineCount { headerIdx = i + 1 break } } } + + lastRowFound := false for i := len(data) - 1; i >= 0; i-- { c := data[i] - if c == 'n' { - footerIdx = i + 1 - break + if c == '\n' { + if lastRowFound { + footerIdx = i + 1 + break + } + lastRowFound = true } } @@ -101,7 +185,7 @@ func (v *VerizonCSV) trimHeadersAndFooters() { } func ProcessVerizonCsvToObjects[K any](v *VerizonCSV, mappingFunction func([]string) *K, filterFunction func([]string) bool) *[]*K { - ret := make([]*K, 0, 1000) + ret := make([]*K, 0, 10000) doc := csv.NewReader(v.TabularData) var filter func([]string) bool @@ -123,3 +207,11 @@ func ProcessVerizonCsvToObjects[K any](v *VerizonCSV, mappingFunction func([]str return &ret } + +func (call VerizonCallLine) ToQueryBlock() string { + return fmt.Sprintf("('%s','%s',%d,'%s')", call.WirelessNumber, call.CallDate, call.CallMinutes, call.OtherNumber) +} + +func (sms VerizonSMSLine) ToQueryBlock() string { + return fmt.Sprintf("('%s','%s','%s')", sms.WirelessNumber, sms.MsgDateTime, sms.OtherNumber) +}