Merge pull request #20 from AlfHou/feat/drivvo-import

Add importer for Drivvo
This commit is contained in:
Alf Sebastian Houge
2023-01-28 22:51:32 +01:00
committed by GitHub
8 changed files with 589 additions and 148 deletions

View File

@@ -2,6 +2,7 @@ package controllers
import (
"net/http"
"strconv"
"github.com/akhilrex/hammond/service"
"github.com/gin-gonic/gin"
@@ -9,6 +10,7 @@ import (
func RegisteImportController(router *gin.RouterGroup) {
router.POST("/import/fuelly", fuellyImport)
router.POST("/import/drivvo", drivvoImport)
}
func fuellyImport(c *gin.Context) {
@@ -24,3 +26,28 @@ func fuellyImport(c *gin.Context) {
}
c.JSON(http.StatusOK, gin.H{})
}
func drivvoImport(c *gin.Context) {
bytes, err := getFileBytes(c, "file")
if err != nil {
c.JSON(http.StatusUnprocessableEntity, err)
return
}
vehicleId := c.PostForm("vehicleID")
if vehicleId == "" {
c.JSON(http.StatusUnprocessableEntity, "Missing Vehicle ID")
return
}
importLocation, err := strconv.ParseBool(c.PostForm("importLocation"))
if err != nil {
c.JSON(http.StatusUnprocessableEntity, "Please include importLocation option.")
return
}
errors := service.DrivvoImport(bytes, c.MustGet("userId").(string), vehicleId, importLocation)
if len(errors) > 0 {
c.JSON(http.StatusUnprocessableEntity, gin.H{"errors": errors})
return
}
c.JSON(http.StatusOK, gin.H{})
}

View File

@@ -0,0 +1,142 @@
package service
import (
"bytes"
"encoding/csv"
"fmt"
"strconv"
"strings"
"time"
"github.com/akhilrex/hammond/db"
)
func DrivvoParseExpenses(content []byte, user *db.User, vehicle *db.Vehicle) ([]db.Expense, []string) {
expenseReader := csv.NewReader(bytes.NewReader(content))
expenseReader.Comment = '#'
// Read headers (there is a trailing comma at the end, that's why we have to read the first line)
expenseReader.Read()
expenseReader.FieldsPerRecord = 6
expenseRecords, err := expenseReader.ReadAll()
var errors []string
if err != nil {
errors = append(errors, err.Error())
println(err.Error())
return nil, errors
}
var expenses []db.Expense
for index, record := range expenseRecords {
date, err := time.Parse("2006-01-02 15:04:05", record[1])
if err != nil {
errors = append(errors, "Found an invalid date/time at service/expense row "+strconv.Itoa(index+1))
}
totalCost, err := strconv.ParseFloat(record[2], 32)
if err != nil {
errors = append(errors, "Found and invalid total cost at service/expense row "+strconv.Itoa(index+1))
}
odometer, err := strconv.Atoi(record[0])
if err != nil {
errors = append(errors, "Found an invalid odometer reading at service/expense row "+strconv.Itoa(index+1))
}
notes := fmt.Sprintf("Location: %s\nNotes: %s\n", record[4], record[5])
expenses = append(expenses, db.Expense{
UserID: user.ID,
VehicleID: vehicle.ID,
Date: date,
OdoReading: odometer,
Amount: float32(totalCost),
ExpenseType: record[3],
Currency: user.Currency,
DistanceUnit: user.DistanceUnit,
Comments: notes,
Source: "Drivvo",
})
}
return expenses, errors
}
func DrivvoParseRefuelings(content []byte, user *db.User, vehicle *db.Vehicle, importLocation bool) ([]db.Fillup, []string) {
refuelingReader := csv.NewReader(bytes.NewReader(content))
refuelingReader.Comment = '#'
refuelingRecords, err := refuelingReader.ReadAll()
var errors []string
if err != nil {
errors = append(errors, err.Error())
println(err.Error())
return nil, errors
}
var fillups []db.Fillup
for index, record := range refuelingRecords {
// Skip column titles
if index == 0 {
continue
}
date, err := time.Parse("2006-01-02 15:04:05", record[1])
if err != nil {
errors = append(errors, "Found an invalid date/time at refuel row "+strconv.Itoa(index+1))
}
totalCost, err := strconv.ParseFloat(record[4], 32)
if err != nil {
errors = append(errors, "Found and invalid total cost at refuel row "+strconv.Itoa(index+1))
}
odometer, err := strconv.Atoi(record[0])
if err != nil {
errors = append(errors, "Found an invalid odometer reading at refuel row "+strconv.Itoa(index+1))
}
location := ""
if importLocation {
location = record[17]
}
pricePerUnit, err := strconv.ParseFloat(record[3], 32)
if err != nil {
unit := strings.ToLower(db.FuelUnitDetails[vehicle.FuelUnit].Key)
errors = append(errors, fmt.Sprintf("Found an invalid cost per %s at refuel row %d", unit, index+1))
}
quantity, err := strconv.ParseFloat(record[5], 32)
if err != nil {
errors = append(errors, "Found an invalid quantity at refuel row "+strconv.Itoa(index+1))
}
isTankFull := record[6] == "Yes"
// Unfortunatly, drivvo doesn't expose this info in their export
fal := false
notes := fmt.Sprintf("Reason: %s\nNotes: %s\nFuel: %s\n", record[18], record[19], record[2])
fillups = append(fillups, db.Fillup{
VehicleID: vehicle.ID,
UserID: user.ID,
Date: date,
HasMissedFillup: &fal,
IsTankFull: &isTankFull,
FuelQuantity: float32(quantity),
PerUnitPrice: float32(pricePerUnit),
FillingStation: location,
OdoReading: odometer,
TotalAmount: float32(totalCost),
FuelUnit: vehicle.FuelUnit,
Currency: user.Currency,
DistanceUnit: user.DistanceUnit,
Comments: notes,
Source: "Drivvo",
})
}
return fillups, errors
}

View File

@@ -0,0 +1,140 @@
package service
import (
"bytes"
"encoding/csv"
"fmt"
"strconv"
"time"
"github.com/akhilrex/hammond/db"
"github.com/leekchan/accounting"
)
func FuellyParseAll(content []byte, userId string) ([]db.Fillup, []db.Expense, []string) {
stream := bytes.NewReader(content)
reader := csv.NewReader(stream)
records, err := reader.ReadAll()
var errors []string
user, err := GetUserById(userId)
if err != nil {
errors = append(errors, err.Error())
return nil, nil, errors
}
vehicles, err := GetUserVehicles(userId)
if err != nil {
errors = append(errors, err.Error())
return nil, nil, errors
}
if err != nil {
errors = append(errors, err.Error())
return nil, nil, errors
}
var vehicleMap map[string]db.Vehicle = make(map[string]db.Vehicle)
for _, vehicle := range *vehicles {
vehicleMap[vehicle.Nickname] = vehicle
}
var fillups []db.Fillup
var expenses []db.Expense
layout := "2006-01-02 15:04"
altLayout := "2006-01-02 3:04 PM"
for index, record := range records {
if index == 0 {
continue
}
var vehicle db.Vehicle
var ok bool
if vehicle, ok = vehicleMap[record[4]]; !ok {
errors = append(errors, "Found an unmapped vehicle entry at row "+strconv.Itoa(index+1))
}
dateStr := record[2] + " " + record[3]
date, err := time.Parse(layout, dateStr)
if err != nil {
date, err = time.Parse(altLayout, dateStr)
}
if err != nil {
errors = append(errors, "Found an invalid date/time at row "+strconv.Itoa(index+1))
}
totalCostStr := accounting.UnformatNumber(record[9], 3, user.Currency)
totalCost64, err := strconv.ParseFloat(totalCostStr, 32)
if err != nil {
errors = append(errors, "Found an invalid total cost at row "+strconv.Itoa(index+1))
}
totalCost := float32(totalCost64)
odoStr := accounting.UnformatNumber(record[5], 0, user.Currency)
odoreading, err := strconv.Atoi(odoStr)
if err != nil {
errors = append(errors, "Found an invalid odo reading at row "+strconv.Itoa(index+1))
}
location := record[12]
//Create Fillup
if record[0] == "Gas" {
rateStr := accounting.UnformatNumber(record[7], 3, user.Currency)
ratet64, err := strconv.ParseFloat(rateStr, 32)
if err != nil {
errors = append(errors, "Found an invalid cost per gallon at row "+strconv.Itoa(index+1))
}
rate := float32(ratet64)
quantity64, err := strconv.ParseFloat(record[8], 32)
if err != nil {
errors = append(errors, "Found an invalid quantity at row "+strconv.Itoa(index+1))
}
quantity := float32(quantity64)
notes := fmt.Sprintf("Octane:%s\nGas Brand:%s\nLocation%s\nTags:%s\nPayment Type:%s\nTire Pressure:%s\nNotes:%s\nMPG:%s",
record[10], record[11], record[12], record[13], record[14], record[15], record[16], record[1],
)
isTankFull := record[6] == "Full"
fal := false
fillups = append(fillups, db.Fillup{
VehicleID: vehicle.ID,
FuelUnit: vehicle.FuelUnit,
FuelQuantity: quantity,
PerUnitPrice: rate,
TotalAmount: totalCost,
OdoReading: odoreading,
IsTankFull: &isTankFull,
Comments: notes,
FillingStation: location,
HasMissedFillup: &fal,
UserID: userId,
Date: date,
Currency: user.Currency,
DistanceUnit: user.DistanceUnit,
Source: "Fuelly",
})
}
if record[0] == "Service" {
notes := fmt.Sprintf("Tags:%s\nPayment Type:%s\nNotes:%s",
record[13], record[14], record[16],
)
expenses = append(expenses, db.Expense{
VehicleID: vehicle.ID,
Amount: totalCost,
OdoReading: odoreading,
Comments: notes,
ExpenseType: record[17],
UserID: userId,
Currency: user.Currency,
Date: date,
DistanceUnit: user.DistanceUnit,
Source: "Fuelly",
})
}
}
return fillups, expenses, errors
}

View File

@@ -2,144 +2,12 @@ package service
import (
"bytes"
"encoding/csv"
"fmt"
"strconv"
"time"
"github.com/akhilrex/hammond/db"
"github.com/leekchan/accounting"
)
func FuellyImport(content []byte, userId string) []string {
stream := bytes.NewReader(content)
reader := csv.NewReader(stream)
records, err := reader.ReadAll()
func WriteToDB(fillups []db.Fillup, expenses []db.Expense) []string {
var errors []string
if err != nil {
errors = append(errors, err.Error())
return errors
}
vehicles, err := GetUserVehicles(userId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
user, err := GetUserById(userId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
var vehicleMap map[string]db.Vehicle = make(map[string]db.Vehicle)
for _, vehicle := range *vehicles {
vehicleMap[vehicle.Nickname] = vehicle
}
var fillups []db.Fillup
var expenses []db.Expense
layout := "2006-01-02 15:04"
altLayout := "2006-01-02 3:04 PM"
for index, record := range records {
if index == 0 {
continue
}
var vehicle db.Vehicle
var ok bool
if vehicle, ok = vehicleMap[record[4]]; !ok {
errors = append(errors, "Found an unmapped vehicle entry at row "+strconv.Itoa(index+1))
}
dateStr := record[2] + " " + record[3]
date, err := time.Parse(layout, dateStr)
if err != nil {
date, err = time.Parse(altLayout, dateStr)
}
if err != nil {
errors = append(errors, "Found an invalid date/time at row "+strconv.Itoa(index+1))
}
totalCostStr := accounting.UnformatNumber(record[9], 3, user.Currency)
totalCost64, err := strconv.ParseFloat(totalCostStr, 32)
if err != nil {
errors = append(errors, "Found an invalid total cost at row "+strconv.Itoa(index+1))
}
totalCost := float32(totalCost64)
odoStr := accounting.UnformatNumber(record[5], 0, user.Currency)
odoreading, err := strconv.Atoi(odoStr)
if err != nil {
errors = append(errors, "Found an invalid odo reading at row "+strconv.Itoa(index+1))
}
location := record[12]
//Create Fillup
if record[0] == "Gas" {
rateStr := accounting.UnformatNumber(record[7], 3, user.Currency)
ratet64, err := strconv.ParseFloat(rateStr, 32)
if err != nil {
errors = append(errors, "Found an invalid cost per gallon at row "+strconv.Itoa(index+1))
}
rate := float32(ratet64)
quantity64, err := strconv.ParseFloat(record[8], 32)
if err != nil {
errors = append(errors, "Found an invalid quantity at row "+strconv.Itoa(index+1))
}
quantity := float32(quantity64)
notes := fmt.Sprintf("Octane:%s\nGas Brand:%s\nLocation%s\nTags:%s\nPayment Type:%s\nTire Pressure:%s\nNotes:%s\nMPG:%s",
record[10], record[11], record[12], record[13], record[14], record[15], record[16], record[1],
)
isTankFull := record[6] == "Full"
fal := false
fillups = append(fillups, db.Fillup{
VehicleID: vehicle.ID,
FuelUnit: vehicle.FuelUnit,
FuelQuantity: quantity,
PerUnitPrice: rate,
TotalAmount: totalCost,
OdoReading: odoreading,
IsTankFull: &isTankFull,
Comments: notes,
FillingStation: location,
HasMissedFillup: &fal,
UserID: userId,
Date: date,
Currency: user.Currency,
DistanceUnit: user.DistanceUnit,
Source: "Fuelly",
})
}
if record[0] == "Service" {
notes := fmt.Sprintf("Tags:%s\nPayment Type:%s\nNotes:%s",
record[13], record[14], record[16],
)
expenses = append(expenses, db.Expense{
VehicleID: vehicle.ID,
Amount: totalCost,
OdoReading: odoreading,
Comments: notes,
ExpenseType: record[17],
UserID: userId,
Currency: user.Currency,
Date: date,
DistanceUnit: user.DistanceUnit,
Source: "Fuelly",
})
}
}
if len(errors) != 0 {
return errors
}
tx := db.DB.Begin()
defer func() {
if r := recover(); r != nil {
@@ -150,19 +18,90 @@ func FuellyImport(content []byte, userId string) []string {
errors = append(errors, err.Error())
return errors
}
if err := tx.Create(&fillups).Error; err != nil {
tx.Rollback()
errors = append(errors, err.Error())
return errors
if fillups != nil {
if err := tx.Create(&fillups).Error; err != nil {
tx.Rollback()
errors = append(errors, err.Error())
return errors
}
}
if err := tx.Create(&expenses).Error; err != nil {
tx.Rollback()
errors = append(errors, err.Error())
return errors
if expenses != nil {
if err := tx.Create(&expenses).Error; err != nil {
tx.Rollback()
errors = append(errors, err.Error())
return errors
}
}
err = tx.Commit().Error
err := tx.Commit().Error
if err != nil {
errors = append(errors, err.Error())
}
return errors
}
func DrivvoImport(content []byte, userId string, vehicleId string, importLocation bool) []string {
var errors []string
user, err := GetUserById(userId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
vehicle, err := GetVehicleById(vehicleId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
endParseIndex := bytes.Index(content, []byte("#Income"))
if endParseIndex == -1 {
endParseIndex = bytes.Index(content, []byte("#Route"))
if endParseIndex == -1 {
endParseIndex = len(content)
}
}
serviceEndIndex := bytes.Index(content, []byte("#Expense"))
if serviceEndIndex == -1 {
serviceEndIndex = endParseIndex
}
refuelEndIndex := bytes.Index(content, []byte("#Service"))
if refuelEndIndex == -1 {
refuelEndIndex = serviceEndIndex
}
var fillups []db.Fillup
fillups, errors = DrivvoParseRefuelings(content[:refuelEndIndex], user, vehicle, importLocation)
var allExpenses []db.Expense
services, parseErrors := DrivvoParseExpenses(content[refuelEndIndex:serviceEndIndex], user, vehicle)
if parseErrors != nil {
errors = append(errors, parseErrors...)
}
allExpenses = append(allExpenses, services...)
expenses, parseErrors := DrivvoParseExpenses(content[serviceEndIndex:endParseIndex], user, vehicle)
if parseErrors != nil {
errors = append(errors, parseErrors...)
}
allExpenses = append(allExpenses, expenses...)
if len(errors) != 0 {
return errors
}
return WriteToDB(fillups, allExpenses)
}
func FuellyImport(content []byte, userId string) []string {
fillups, expenses, errors := FuellyParseAll(content, userId)
if len(errors) != 0 {
return errors
}
return WriteToDB(fillups, expenses)
}

View File

@@ -410,6 +410,15 @@ export default [
},
props: (route) => ({ user: store.state.auth.currentUser || {} }),
},
{
path: '/import/drivvo',
name: 'import-drivvo',
component: () => lazyLoadView(import('@views/import-drivvo.vue')),
meta: {
authRequired: true,
},
props: (route) => ({ user: store.state.auth.currentUser || {} }),
},
{
path: '/logout',
name: 'logout',

View File

@@ -0,0 +1,172 @@
<script>
import Layout from '@layouts/main.vue'
import { mapState } from 'vuex'
import axios from 'axios'
export default {
page: {
title: 'Import Drivvo',
meta: [{ name: 'description', content: 'The Import Drivvo page.' }],
},
components: { Layout },
props: {
user: {
type: Object,
required: true,
},
},
data: function() {
return {
myVehicles: [],
file: null,
selectedVehicle: null,
tryingToCreate: false,
errors: [],
importLocation: true,
}
},
computed: {
...mapState('utils', ['isMobile']),
...mapState('vehicles', ['vehicles']),
uploadButtonLabel() {
if (this.isMobile) {
if (this.file == null) {
return 'Choose Photo'
} else {
return ''
}
} else {
if (this.file == null) {
return 'Choose CSV'
} else {
return ''
}
}
},
},
mounted() {
this.myVehicles = this.vehicles
},
methods: {
importDrivvo() {
console.log('Import from drivvo')
if (this.file == null) {
return
}
this.tryingToCreate = true
this.errorMessage = ''
const formData = new FormData()
formData.append('vehicleID', this.selectedVehicle)
formData.append('importLocation', this.importLocation)
formData.append('file', this.file, this.file.name)
axios
.post(`/api/import/drivvo`, formData)
.then((data) => {
this.$buefy.toast.open({
message: 'Data Imported Successfully',
type: 'is-success',
duration: 3000,
})
this.file = null
setTimeout(() => this.$router.push({ name: 'home' }), 1000)
})
.catch((ex) => {
this.$buefy.toast.open({
duration: 5000,
message: 'There was some issue with importing the file. Please check the error message',
position: 'is-bottom',
type: 'is-danger',
})
if (ex.response && ex.response.data.errors) {
this.errors = ex.response.data.errors
}
})
.finally(() => {
this.tryingToCreate = false
})
},
},
}
</script>
<template>
<Layout>
<div class="columns box">
<div class="column">
<h1 class="title">Import from Drivvo</h1>
</div>
</div>
<br />
<div class="columns">
<div class="column">
<p class="subtitle"> Steps to import data from Drivvo</p>
<ol>
<li>Export your data from Drivvo in the CSV format.</li>
<li>Select the vehicle the exported data is for. You may need to create the vehicle in Hammond first if you haven't already done so</li>
<li
>Make sure that the <u>Currency</u> and <u>Distance Unit</u> are set correctly in Hammond. Drivvo does not include this information in
their export, instead Hammond will use the values set for the user.</li
>
<li>Similiarly, make sure that the <u>Fuel Unit</u> and <u>Fuel Type</u> are correctly set in the Vehicle.</li>
<li>Once you have checked all these points, select the vehicle and import the CSV below.</li>
<li><b>Make sure that you do not import the file again as that will create repeat entries.</b></li>
</ol>
</div>
</div>
<p
><b>PS:</b> If you have <em>'income'</em> and <em>'trips'</em> in your export, they will not be imported to Hammond. The fields
<em>'Second fuel'</em> and <em>'Third fuel'</em> are are are also ignored as the use case for these is not understood by us. If you have a use
case for this, please open a issue on
<a href="https://github.com/akhilrex/hammond/issues">issue tracker</a>
</p>
<div class="section box">
<div class="columns is-multiline">
<div class="column is-full"> <p class="subtitle">Choose the vehicle, then select the Drivvo CSV and press the import button.</p></div>
<div class="column is-full is-flex is-align-content-center">
<form @submit.prevent="importDrivvo">
<div class="columns">
<div class="column">
<b-field label="Vehicle" label-position="on-border">
<b-select v-model="selectedVehicle" placeholder="Select Vehicle" required>
<option v-for="vehicle in myVehicles" :key="vehicle.id" :value="vehicle.id">{{ vehicle.nickname }}</option>
</b-select>
</b-field>
</div>
<div class="column">
<b-field>
<b-tooltip label="Whether to import the location for fillups and services or not." multilined>
<b-checkbox v-model="importLocation">Import Location?</b-checkbox>
</b-tooltip>
</b-field>
</div>
<div class="column">
<b-field class="file is-primary" :class="{ 'has-name': !!file }">
<b-upload v-model="file" class="file-label" accept=".csv" required>
<span class="file-cta">
<b-icon class="file-icon" icon="upload"></b-icon>
<span class="file-label">{{ uploadButtonLabel }}</span>
</span>
<span v-if="file" class="file-name" :class="isMobile ? 'file-name-mobile' : 'file-name-desktop'">
{{ file.name }}
</span>
</b-upload>
</b-field>
</div>
<div class="column">
<b-button tag="input" native-type="submit" :disabled="tryingToCreate" type="is-primary" value="Upload File" class="control">
Import
</b-button>
</div></div
>
</form>
</div>
</div>
</div>
<b-message v-if="errors.length" type="is-danger">
<ul>
<li v-for="error in errors" :key="error">{{ error }}</li>
</ul>
</b-message>
</Layout>
</template>

View File

@@ -58,6 +58,7 @@ export default {
duration: 3000,
})
this.file = null
setTimeout(() => this.$router.push({ name: 'home' }), 1000)
})
.catch((ex) => {
this.$buefy.toast.open({
@@ -108,7 +109,7 @@ export default {
<div class="columns"
><div class="column">
<b-field class="file is-primary" :class="{ 'has-name': !!file }">
<b-upload v-model="file" class="file-label" accept=".csv">
<b-upload v-model="file" class="file-label" accept=".csv" required>
<span class="file-cta">
<b-icon class="file-icon" icon="upload"></b-icon>
<span class="file-label">{{ uploadButtonLabel }}</span>

View File

@@ -26,11 +26,22 @@ export default {
>
<br />
<div class="columns">
<div class="box column is-one-third" to="/import-fuelly">
<h1 class="title">Fuelly</h1>
<p>{{ $t('importcsv', { 'name': 'Fuelly' }) }}</p>
<br />
<b-button type="is-primary" tag="router-link" to="/import/fuelly">{{ $t('import') }}</b-button>
<div class="column is-one-third">
<div class="box">
<h1 class="title">Fuelly</h1>
<p>If you have been using Fuelly to store your vehicle data, export the CSV file from Fuelly and click here to import.</p>
<br />
<b-button type="is-primary" tag="router-link" to="/import/fuelly">{{ $t('import') }}</b-button>
</div>
</div>
<div class="column is-one-third" to="/import-fuelly">
<div class="box">
<h1 class="title">Drivvo</h1>
<p>{{ $t('importcsv', { 'name': 'Fuelly' }) }}</p>
<br />
<b-button type="is-primary" tag="router-link" to="/import/drivvo">{{ $t('import') }}</b-button>
</div>
</div>
</div>
</Layout>