Added Blocker (for turning objects into 'blocks' of query text) interface in database-primitives.go

along with an ExecuteString method for *ConnectorGenerics that will take a file name and run a concatenated query.
This VASTLY improves insert times.

Added a hackey pseudoinit function into Interconnect.go in order to get access to a database without all the headache and time of recreating all the tables.

Updated verizon.go to use the new query functions

src/db/database-primitives.go

@@ -25,6 +25,10 @@ const (
                               ConnectorGeneric
 ======================================================================================*/
 
+type Blocker interface {
+	ToQueryBlock() string
+}
+
 type ConnectorGeneric struct {
 	cachedConnection *sql.DB
 }
@@ -54,6 +58,21 @@ func (c *ConnectorGeneric) ExecuteSqlScript(runner *sqlScriptRunner) {
 	}
 }
 
+func (c *ConnectorGeneric) ExecuteString(dbName string, payload string) {
+	c.startConnection(dbName)
+	defer c.returnConnection()
+
+	//log.Println("==================================================================================================")
+	//log.Println(payload)
+	//log.Println("==================================================================================================")
+
+	_, err := c.cachedConnection.Exec(payload)
+	if err != nil {
+		log.Printf("Query involved in error: %s\n", payload)
+		log.Panic(err)
+	}
+}
+
 // startConnection this initializes, caches, and returns a connection for the desired database. If a connection already exists,
 // it will be terminated and a new connection will be opened.
 func (c *ConnectorGeneric) startConnection(dataBase string) {
@@ -104,6 +123,7 @@ func (c *ConnectorGeneric) CreateTables() {
 		NewRunner("create-insight-timeEntry-table.sql", InsightDatabaseName),
 		NewRunner("create-mercury-picturePatterns-table.sql", MercuryDatabaseName),
 		NewRunner("create-mercury-arReports-table.sql", MercuryDatabaseName),
+		NewRunner("create-mercury-telecomVoice-table.sql", MercuryDatabaseName),
 	}
 
 	for _, runner := range tableCreationRunners {
@@ -172,6 +192,20 @@ func BulkUpdate[K any](connector *ConnectorGeneric, dbName string, updateScript
 	}
 }
 
+func BlockUpdate[K Blocker](connector *ConnectorGeneric, dbName string, updateScript string, items *[]*K) {
+	template := loadSqlFile(updateScript)
+	buff := strings.Builder{}
+	l := len(*items)
+	for i, item := range *items {
+		buff.WriteString((*item).ToQueryBlock())
+		if i < l-1 {
+			buff.WriteByte(',')
+		}
+	}
+	query := fmt.Sprintf(*template, buff.String())
+	connector.ExecuteString(dbName, query)
+}
+
 func NewRunner(scriptName string, databaseName string) *sqlScriptRunner {
 	return &sqlScriptRunner{scriptName, databaseName}
 }

src/mercury.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"fmt"
 	"log"
+	"mercury/src/db"
 	"mercury/src/finance"
 	"mercury/src/mercury"
-	"mercury/src/telecom"
 	"os"
 	"time"
 )
@@ -24,17 +23,10 @@ func main() {
 }
 
 func test() {
-
+	icx := mercury.NewInterconnect()
-	csv := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/call-log-6-7-2022.csv")
+	icx.PseudoInit()
-	results := telecom.ProcessVerizonCsvToObjects(csv, telecom.RowToVerizonCallLine, func(row []string) bool {
+	icx.InsightDBConnector.ExecuteSqlScript(db.NewRunner("create-mercury-telecomVoice-table.sql", db.MercuryDatabaseName))
-		if row[5] == "Voice" {
+	icx.UpdateVerizonReports()
-			return true
-		} else {
-			return false
-		}
-
-	})
-	fmt.Println(len(*results))
 }
 
 func updateInsightData() {

src/mercury/Interconnect.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"mercury/src/db"
 	"mercury/src/projectInsight"
+	"mercury/src/telecom"
 	"time"
 )
 
@@ -23,6 +24,11 @@ func NewInterconnect() *Interconnect {
 	return &connect
 }
 
+// PseudoInit This is a stupid idea in order to fix the fact that we obliterate tables left-and righta t processing time
+func (ic *Interconnect) PseudoInit() {
+	ic.InsightDBConnector = projectInsight.NewDBConnection()
+}
+
 func (ic *Interconnect) Init() {
 	ic.InsightDBConnector = projectInsight.NewDBConnection()
 	ic.InsightDBConnector.ConnectorGeneric.ProcessClarityScripts()
@@ -59,6 +65,19 @@ func (ic *Interconnect) UpdateTimeEntries() {
 	ic.InsightDBConnector.ExecuteSqlScript(db.NewRunner("create-insight-contribution-table.sql", "insight"))
 }
 
+func (ic *Interconnect) UpdateVerizonReports() {
+	log.Println("Reading CSV Verizon Reports")
+	callCSV := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/verizon-call.csv")
+	smsCSV := telecom.NewVerizonCSV("/home/dtookey/work/clarity-reporting/telecom/verizon-sms.csv")
+	callObjects := telecom.ProcessVerizonCsvToObjects(callCSV, telecom.RowToVerizonCallLine, telecom.VerizonCallDataFilter)
+	smsObjects := telecom.ProcessVerizonCsvToObjects(smsCSV, telecom.RowToVerizonSMSLine, telecom.VerizonSMSDataFilter)
+	log.Println("Beginning voice data transfer.")
+	db.BlockUpdate[telecom.VerizonCallLine](ic.InsightDBConnector.ConnectorGeneric, db.MercuryDatabaseName, "update-mercury-voiceCallLine.sql", callObjects)
+	log.Println("Completed...\nBeginning sms data transfer.")
+	db.BlockUpdate[telecom.VerizonSMSLine](ic.InsightDBConnector.ConnectorGeneric, db.MercuryDatabaseName, "update-mercury-smsLine.sql", smsObjects)
+	log.Println("Completed...")
+}
+
 //</editor-fold>
 
 //<editor-fold name="UtilityFunctions">

src/sql/create-mercury-telecomVoice-table.sql

@@ -0,0 +1,23 @@
+DROP TABLE IF EXISTS telecom_voice;
+DROP TABLE IF EXISTS telecom_sms;
+
+CREATE TABLE mercury.telecom_voice
+(
+    wireless_number VARCHAR(12),
+    call_date       date,
+    call_minutes    int,
+    other_number    varchar(12)
+);
+
+CREATE TABLE mercury.telecom_sms
+(
+    wireless_number VARCHAR(12),
+    msg_datetime    datetime,
+    other_number    VARCHAR(12),
+    count_row       int default 1
+);
+
+
+
+
+

src/sql/update-mercury-smsLine.sql

@@ -0,0 +1,3 @@
+
+INSERT INTO mercury.telecom_sms (wireless_number, msg_datetime, other_number)
+VALUES %s;

src/sql/update-mercury-voiceCallLine.sql

@@ -0,0 +1,3 @@
+
+INSERT INTO mercury.telecom_voice (wireless_number, call_date, call_minutes, other_number)
+VALUES %s;

src/telecom/verizon.go

@@ -2,22 +2,32 @@ package telecom
 
 import (
 	"bytes"
+	"database/sql"
 	"encoding/csv"
+	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"strconv"
+	"strings"
+	"time"
 )
 
 /*
 First step is building the reports on verizon's website. Below are the columns (and hopefully categories) that you need
 to completely recreate the report.
 
-
+VerizonCallLine (report structure):
 Wireless number [mandatory]
-"User name" [Contact Information]
 Date [Voice Usage]
 Minutes [voice usage]
 Number [voice usage]
+Usage Type [mandatory]
+
+VerizonSMSLine (report structure):
+Wireless number [mandatory]
+Message date/time [Messaging usage]
+to/from wireless number [Messaging usage]
 
 
 The report comes as a comma/LF formatted CSV. It has 13 lines of useless header data and 1 line of useless total data
@@ -25,6 +35,36 @@ that must be trimmed off to shove everything through a struct factory.
 
 */
 
+var VerizonCallLineDBMappingFunction = func(s *sql.Stmt, item *VerizonCallLine) {
+	_, err := s.Exec(
+		item.WirelessNumber,
+		item.CallDate,
+		item.CallMinutes,
+		item.OtherNumber,
+	)
+	if err != nil {
+		log.Printf("%#v\n", s)
+		log.Panic(err)
+	}
+}
+
+var VerizonSmsLineDBMappingFunction = func(s *sql.Stmt, item *VerizonSMSLine) {
+	_, err := s.Exec(
+		item.WirelessNumber,
+		item.MsgDateTime,
+		item.OtherNumber,
+	)
+	if err != nil {
+		log.Printf("%#v\n", s)
+		log.Panic(err)
+	}
+}
+
+const (
+	verizonCallHeaderLineCount = 14
+	verizonSMSHeaderLineCount  = 14
+)
+
 type (
 	VerizonCSV struct {
 		trimmed     bool
@@ -34,23 +74,62 @@ type (
 
 	VerizonCallLine struct {
 		WirelessNumber string
-		UserName       string
 		CallDate       string
 		CallMinutes    int
 		OtherNumber    string
-		UsageCategory  string
+	}
+
+	VerizonSMSLine struct {
+		WirelessNumber string
+		MsgDateTime    string
+		OtherNumber    string
 	}
 )
 
 func RowToVerizonCallLine(row []string) *VerizonCallLine {
-	minutes, err := strconv.Atoi(row[3])
+	minutes, err := strconv.Atoi(row[2])
 	if err != nil {
+		fmt.Printf("%#v\n", row)
 		panic(err)
 	}
-	line := VerizonCallLine{row[0], row[1], row[2], minutes, row[4], row[5]}
+	line := VerizonCallLine{row[0], convertDateField(row[1]), minutes, convertSolidPhoneNumberToDashes(row[3])}
 	return &line
 }
 
+func RowToVerizonSMSLine(row []string) *VerizonSMSLine {
+	line := VerizonSMSLine{row[0], convertDateTimeField(row[1]), row[2]}
+	return &line
+}
+
+func VerizonCallDataFilter(row []string) bool {
+	return len(row[2]) > 0
+}
+
+func VerizonSMSDataFilter(row []string) bool {
+	return len(row[2]) > 0
+}
+
+func convertDateField(d string) string {
+	parts := strings.Split(d, "/")
+	return fmt.Sprintf("%s-%s-%s", parts[2], parts[0], parts[1])
+}
+
+func convertDateTimeField(dt string) string {
+	dateTime, err := time.Parse("1/2/06 3:04 PM", dt)
+	if err != nil {
+		panic(err)
+	}
+
+	return dateTime.Add(-4 * time.Hour).Format("2006-01-02 15:04:05.000000")
+}
+
+func convertSolidPhoneNumberToDashes(str string) string {
+	if len(str) < 10 {
+		return ""
+	}
+	return fmt.Sprintf("%s-%s-%s", str[0:3], str[3:6], str[6:10])
+}
+
 func NewVerizonCSV(pathlike string) *VerizonCSV {
 	csv := VerizonCSV{trimmed: false}
 	file, err := os.OpenFile(pathlike, os.O_RDONLY, 777)
@@ -81,17 +160,22 @@ func (v *VerizonCSV) trimHeadersAndFooters() {
 	for i, c := range data {
 		if c == '\n' {
 			lineCount += 1
-			if lineCount == 13 {
+			if lineCount == verizonCallHeaderLineCount {
 				headerIdx = i + 1
 				break
 			}
 		}
 	}
+
+	lastRowFound := false
 	for i := len(data) - 1; i >= 0; i-- {
 		c := data[i]
-		if c == 'n' {
+		if c == '\n' {
-			footerIdx = i + 1
+			if lastRowFound {
-			break
+				footerIdx = i + 1
+				break
+			}
+			lastRowFound = true
 		}
 	}
 
@@ -101,7 +185,7 @@ func (v *VerizonCSV) trimHeadersAndFooters() {
 }
 
 func ProcessVerizonCsvToObjects[K any](v *VerizonCSV, mappingFunction func([]string) *K, filterFunction func([]string) bool) *[]*K {
-	ret := make([]*K, 0, 1000)
+	ret := make([]*K, 0, 10000)
 	doc := csv.NewReader(v.TabularData)
 
 	var filter func([]string) bool
@@ -123,3 +207,11 @@ func ProcessVerizonCsvToObjects[K any](v *VerizonCSV, mappingFunction func([]str
 
 	return &ret
 }
+
+func (call VerizonCallLine) ToQueryBlock() string {
+	return fmt.Sprintf("('%s','%s',%d,'%s')", call.WirelessNumber, call.CallDate, call.CallMinutes, call.OtherNumber)
+}
+
+func (sms VerizonSMSLine) ToQueryBlock() string {
+	return fmt.Sprintf("('%s','%s','%s')", sms.WirelessNumber, sms.MsgDateTime, sms.OtherNumber)
+}