gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

started working on migration code.

Browse Source
This commit is contained in:
Jason Kulatunga
2022-05-03 11:50:22 -07:00
parent 97f6564c1e
commit 9d85920f49
9 changed files with 337 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/database/migrations/m20201107210306/device.go
+30
View File
@@ -0,0 +1,30 @@
package m20201107210306
import (
"time"
)
type Device struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt *time.Time
WWN string `json:"wwn" gorm:"primary_key"`
HostId string `json:"host_id"`
DeviceName string `json:"device_name"`
Manufacturer string `json:"manufacturer"`
ModelName string `json:"model_name"`
InterfaceType string `json:"interface_type"`
InterfaceSpeed string `json:"interface_speed"`
SerialNumber string `json:"serial_number"`
Firmware string `json:"firmware"`
RotationSpeed int `json:"rotational_speed"`
Capacity int64 `json:"capacity"`
FormFactor string `json:"form_factor"`
SmartSupport bool `json:"smart_support"`
DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector.
SmartResults []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
}
webapp/backend/pkg/database/migrations/m20201107210306/smart.go
+25
View File
@@ -0,0 +1,25 @@
package m20201107210306
import (
"gorm.io/gorm"
"time"
)
type Smart struct {
gorm.Model
DeviceWWN string `json:"device_wwn"`
Device Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
TestDate time.Time `json:"date"`
SmartStatus string `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
//Metrics
Temp int64 `json:"temp"`
PowerOnHours int64 `json:"power_on_hours"`
PowerCycleCount int64 `json:"power_cycle_count"`
AtaAttributes []SmartAtaAttribute `json:"ata_attributes" gorm:"foreignkey:SmartId"`
NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
}
webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go
+25
View File
@@ -0,0 +1,25 @@
package m20201107210306
import "gorm.io/gorm"
type SmartAtaAttribute struct {
gorm.Model
SmartId int `json:"smart_id"`
Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
AttributeId int `json:"attribute_id"`
Name string `json:"name"`
Value int `json:"value"`
Worst int `json:"worst"`
Threshold int `json:"thresh"`
RawValue int64 `json:"raw_value"`
RawString string `json:"raw_string"`
WhenFailed string `json:"when_failed"`
TransformedValue int64 `json:"transformed_value"`
Status string `gorm:"-" json:"status,omitempty"`
StatusReason string `gorm:"-" json:"status_reason,omitempty"`
FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"`
History []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
}
webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go
+21
View File
@@ -0,0 +1,21 @@
package m20201107210306
import "gorm.io/gorm"
type SmartNvmeAttribute struct {
gorm.Model
SmartId int `json:"smart_id"`
Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
AttributeId string `json:"attribute_id"` //json string from smartctl
Name string `json:"name"`
Value int `json:"value"`
Threshold int `json:"thresh"`
TransformedValue int64 `json:"transformed_value"`
Status string `gorm:"-" json:"status,omitempty"`
StatusReason string `gorm:"-" json:"status_reason,omitempty"`
FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"`
History []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
}
webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go
+21
View File
@@ -0,0 +1,21 @@
package m20201107210306
import "gorm.io/gorm"
type SmartScsiAttribute struct {
gorm.Model
SmartId int `json:"smart_id"`
Smart Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
AttributeId string `json:"attribute_id"` //json string from smartctl
Name string `json:"name"`
Value int `json:"value"`
Threshold int `json:"thresh"`
TransformedValue int64 `json:"transformed_value"`
Status string `gorm:"-" json:"status,omitempty"`
StatusReason string `gorm:"-" json:"status_reason,omitempty"`
FailureRate float64 `gorm:"-" json:"failure_rate,omitempty"`
History []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
}
webapp/backend/pkg/database/scrutiny_repository.go
+13 -2
View File
@@ -57,17 +57,18 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Gorm/SQLite setup
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
//TODO: figure out how to log database queries again.
//Logger: logger
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
return nil, fmt.Errorf("Failed to connect to database! - %v", err)
}
globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
//database.SetLogger()
database.AutoMigrate(&models.Device{})
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// InfluxDB setup
@@ -143,6 +144,16 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
if err != nil {
return nil, err
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// InfluxDB & SQLite migrations
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//database.AutoMigrate(&models.Device{})
err = deviceRepo.Migrate(backgroundContext)
if err != nil {
return nil, err
}
return &deviceRepo, nil
}
webapp/backend/pkg/database/scrutiny_repository_migrations.go
+96
View File
@@ -0,0 +1,96 @@
package database
import (
"context"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/go-gormigrate/gormigrate/v2"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"gorm.io/gorm"
)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// SQLite migrations
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//database.AutoMigrate(&models.Device{})
func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
sr.logger.Infoln("Database migration starting")
m := gormigrate.New(sr.gormClient, gormigrate.DefaultOptions, []*gormigrate.Migration{
{
ID: "20201107210306", // v0.3.13 (pre-influxdb schema). 9fac3c6308dc6cb6cd5bbc43a68cd93e8fb20b87
Migrate: func(tx *gorm.DB) error {
// it's a good practice to copy the struct inside the function,
return tx.AutoMigrate(
&m20201107210306.Device{},
&m20201107210306.Smart{},
&m20201107210306.SmartAtaAttribute{},
&m20201107210306.SmartNvmeAttribute{},
&m20201107210306.SmartNvmeAttribute{},
)
},
Rollback: func(tx *gorm.DB) error {
return tx.Migrator().DropTable(
&m20201107210306.Device{},
&m20201107210306.Smart{},
&m20201107210306.SmartAtaAttribute{},
&m20201107210306.SmartNvmeAttribute{},
&m20201107210306.SmartNvmeAttribute{},
"self_tests",
)
},
},
{
ID: "20220503113100", // backwards compatible - influxdb schema
Migrate: func(tx *gorm.DB) error {
// delete unnecessary table.
err := tx.Migrator().DropTable("self_tests")
if err != nil {
return err
}
//add columns to the Device schema, so we can start adding data to the database & influxdb
err = tx.Migrator().AddColumn(&models.Device{}, "Label") //Label string `json:"label"`
if err != nil {
return err
}
err = tx.Migrator().AddColumn(&models.Device{}, "DeviceStatus") //DeviceStatus pkg.DeviceStatus `json:"device_status"`
if err != nil {
return err
}
//TODO: migrate the data from GORM to influxdb.
return nil
},
},
//{
// ID: "20220503120000", // v0.4.0 - influxdb schema
// Migrate: func(tx *gorm.DB) error {
// // delete unnecessary tables.
// err := tx.Migrator().DropTable(
// &m20201107210306.Smart{},
// &m20201107210306.SmartAtaAttribute{},
// &m20201107210306.SmartNvmeAttribute{},
// &m20201107210306.SmartNvmeAttribute{},
// )
// if err != nil {
// return err
// }
//
// //migrate the device database to the final version
// return tx.AutoMigrate(models.Device{})
// },
//},
})
if err := m.Migrate(); err != nil {
sr.logger.Errorf("Database migration failed with error: %w", err)
return err
}
sr.logger.Infoln("Database migration completed successfully")
return nil
}