Merge pull request #49 from alienp4nda/feat/generic-import

Feat/generic import
This commit is contained in:
Alf Sebastian Houge
2023-04-27 08:51:13 +02:00
committed by GitHub
12 changed files with 623 additions and 61 deletions

View File

@@ -4,6 +4,7 @@ import (
"net/http"
"strconv"
"hammond/models"
"hammond/service"
"github.com/gin-gonic/gin"
@@ -12,6 +13,7 @@ import (
func RegisteImportController(router *gin.RouterGroup) {
router.POST("/import/fuelly", fuellyImport)
router.POST("/import/drivvo", drivvoImport)
router.POST("/import/generic", genericImport)
}
func fuellyImport(c *gin.Context) {
@@ -52,3 +54,21 @@ func drivvoImport(c *gin.Context) {
}
c.JSON(http.StatusOK, gin.H{})
}
func genericImport(c *gin.Context) {
var json models.ImportData
if err := c.ShouldBindJSON(&json); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if json.VehicleId == "" {
c.JSON(http.StatusUnprocessableEntity, "Missing Vehicle ID")
return
}
errors := service.GenericImport(json, c.MustGet("userId").(string))
if len(errors) > 0 {
c.JSON(http.StatusUnprocessableEntity, gin.H{"errors": errors})
return
}
c.JSON(http.StatusOK, gin.H{})
}

22
server/models/import.go Normal file
View File

@@ -0,0 +1,22 @@
package models
type ImportData struct {
Data []ImportFillup `json:"data" binding:"required"`
VehicleId string `json:"vehicleId" binding:"required"`
TimeZone string `json:"timezone" binding:"required"`
}
type ImportFillup struct {
VehicleID string `json:"vehicleId"`
FuelQuantity float32 `json:"fuelQuantity"`
PerUnitPrice float32 `json:"perUnitPrice"`
TotalAmount float32 `json:"totalAmount"`
OdoReading int `json:"odoReading"`
IsTankFull *bool `json:"isTankFull"`
HasMissedFillup *bool `json:"hasMissedFillup"`
Comments string `json:"comments"`
FillingStation string `json:"fillingStation"`
UserID string `json:"userId"`
Date string `json:"date"`
FuelSubType string `json:"fuelSubType"`
}

View File

@@ -0,0 +1,47 @@
package service
import (
"hammond/db"
"hammond/models"
"time"
)
func GenericParseRefuelings(content []models.ImportFillup, user *db.User, vehicle *db.Vehicle, timezone string) ([]db.Fillup, []string) {
var errors []string
var fillups []db.Fillup
dateLayout := "2006-01-02T15:04:05.000Z"
loc, _ := time.LoadLocation(timezone)
for _, record := range content {
date, err := time.ParseInLocation(dateLayout, record.Date, loc)
if err != nil {
date = time.Date(2000, time.December, 0, 0, 0, 0, 0, loc)
}
var missedFillup bool
if record.HasMissedFillup == nil {
missedFillup = false
} else {
missedFillup = *record.HasMissedFillup
}
fillups = append(fillups, db.Fillup{
VehicleID: vehicle.ID,
UserID: user.ID,
Date: date,
IsTankFull: record.IsTankFull,
HasMissedFillup: &missedFillup,
FuelQuantity: float32(record.FuelQuantity),
PerUnitPrice: float32(record.PerUnitPrice),
FillingStation: record.FillingStation,
OdoReading: record.OdoReading,
TotalAmount: float32(record.TotalAmount),
FuelUnit: vehicle.FuelUnit,
Currency: user.Currency,
DistanceUnit: user.DistanceUnit,
Comments: record.Comments,
Source: "Generic Import",
})
}
return fillups, errors
}

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"hammond/db"
"hammond/models"
)
func WriteToDB(fillups []db.Fillup, expenses []db.Expense) []string {
@@ -105,3 +106,27 @@ func FuellyImport(content []byte, userId string) []string {
return WriteToDB(fillups, expenses)
}
func GenericImport(content models.ImportData, userId string) []string {
var errors []string
user, err := GetUserById(userId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
vehicle, err := GetVehicleById(content.VehicleId)
if err != nil {
errors = append(errors, err.Error())
return errors
}
var fillups []db.Fillup
fillups, errors = GenericParseRefuelings(content.Data, user, vehicle, content.TimeZone)
if len(errors) != 0 {
return errors
}
return WriteToDB(fillups, nil)
}

View File

@@ -36,6 +36,7 @@
"node-gyp": "^9.3.1",
"normalize.css": "^8.0.1",
"nprogress": "^0.2.0",
"papaparse": "^5.4.1",
"vue": "^2.6.11",
"vue-chartjs": "^3.5.1",
"vue-i18n": "^8.28.2",

View File

@@ -137,7 +137,7 @@
"dontimportagain": "Achte darauf, dass du die Datei nicht erneut importierst, da dies zu mehrfachen Einträgen führen würde.",
"checkpointsimportcsv": "Wenn du alle diese Punkte überprüft hast kannst du unten die CSV importieren.",
"importhintunits": "Vergewissere dich ebenfalls, dass die <u>Kraftstoffeinheit</u> und der <u>Kraftstofftyp</u> im Fahrzeug richtig eingestellt sind.",
"importhintcurrdist": "Stelle sicher, dass die <u>Währung</u> und die <u>Entfernungseinheit</u> in Hammond korrekt eingestellt sind. Der Import erkennt die Währung nicht automatisch aus der CSV-Datei, sondern verwendet die für den Benutzer eingestellte Währung.",
"importhintcurrdist": "Stelle sicher, dass die <u>Währung</u> und die <u>Entfernungseinheit</u> in Hammond korrekt eingestellt sind. Der Import erkennt die Währung nicht automatisch aus der datei, sondern verwendet die für den Benutzer eingestellte Währung.",
"importhintnickname": "Vergewissere dich, dass der Fahrzeugname in Hammond genau mit dem Namen in der Fuelly-CSV-Datei übereinstimmt, sonst funktioniert der Import nicht.",
"importhintvehiclecreated": "Vergewissere dich, dass du die Fahrzeuge bereits in Hammond erstellt hast.",
"importhintcreatecsv": "Exportiere deine Daten aus {name} im CSV-Format. Die Schritte dazu findest du",

View File

@@ -117,7 +117,9 @@
"price": "Price",
"total": "Total",
"fulltank": "Tank Full",
"partialfillup": "Partial Fillup",
"getafulltank": "Did you get a full tank?",
"tankpartialfull": "Which do you track?",
"by": "By",
"expenses": "Expenses",
"expensetype": "Expense Type",
@@ -130,6 +132,8 @@
"importdatadesc": "Choose from the following options to import data into Hammond",
"import": "Import",
"importcsv": "If you have been using {name} to store your vehicle data, export the CSV file from {name} and click here to import.",
"importgeneric": "Generic Fillups Import",
"importgenericdesc": "Fillups CSV import.",
"choosecsv": "Choose CSV",
"choosephoto": "Choose Photo",
"importsuccessfull": "Data Imported Successfully",
@@ -137,13 +141,15 @@
"importfrom": "Import from {0}",
"stepstoimport": "Steps to import data from {name}",
"choosecsvimport": "Choose the {name} CSV and press the import button.",
"choosedatafile": "Choose the CSV file and then press the import button.",
"dontimportagain": "Make sure that you do not import the file again because that will create repeat entries.",
"checkpointsimportcsv": "Once you have checked all these points, just import the CSV below.",
"importhintunits": "Similiarly, make sure that the <u>Fuel Unit</u> and <u>Fuel Type</u> are correctly set in the Vehicle.",
"importhintcurrdist": "Make sure that the <u>Currency</u> and <u>Distance Unit</u> are set correctly in Hammond. Import will not autodetect Currency from the CSV but use the one set for the user.",
"importhintcurrdist": "Make sure that the <u>Currency</u> and <u>Distance Unit</u> are set correctly in Hammond. Import will not autodetect Currency from the file but use the one set for the user.",
"importhintnickname": "Make sure that the Vehicle nickname in Hammond is exactly the same as the name on Fuelly CSV or the import will not work.",
"importhintvehiclecreated": "Make sure that you have already created the vehicles in Hammond platform.",
"importhintcreatecsv": "Export your data from {name} in the CSV format. Steps to do that can be found",
"importgenerichintdata": "Data must be in CSV format.",
"here": "here",
"unprocessedquickentries": "You have one quick entry to be processed. | You have {0} quick entries pending to be processed.",
"show": "Show",
@@ -178,6 +184,7 @@
"fillingstation": "Filling Station Name",
"comments": "Comments",
"missfillupbefore": "Did you miss the fillup entry before this one?",
"missedfillup": "Missed Fillup",
"fillupdate": "Fillup Date",
"fillupsavedsuccessfully": "Fillup Saved Successfully",
"expensesavedsuccessfully": "Expense Saved Successfully",

View File

@@ -7,6 +7,7 @@ import {
faCheck,
faTimes,
faArrowUp,
faArrowRotateLeft,
faAngleLeft,
faAngleRight,
faCalendar,
@@ -38,6 +39,7 @@ library.add(
faCheck,
faTimes,
faArrowUp,
faArrowRotateLeft,
faAngleLeft,
faAngleRight,
faCalendar,

View File

@@ -419,6 +419,15 @@ export default [
},
props: (route) => ({ user: store.state.auth.currentUser || {} }),
},
{
path: '/import/generic',
name: 'import-generic',
component: () => lazyLoadView(import('@views/import-generic.vue')),
meta: {
authRequired: true,
},
props: (route) => ({ user: store.state.auth.currentUser || {} }),
},
{
path: '/logout',
name: 'logout',

View File

@@ -0,0 +1,7 @@
import ImportGeneric from './import-generic'
describe('@views/import-generic', () => {
it('is a valid view', () => {
expect(ImportGeneric).toBeAViewComponent()
})
})

View File

@@ -0,0 +1,411 @@
<script>
import Layout from '@layouts/main.vue'
import { mapState } from 'vuex'
import axios from 'axios'
import Papa from 'papaparse'
export default {
page: {
title: 'Generic Import',
meta: [{ name: 'description', content: 'The Generic Import page.' }],
},
components: { Layout },
props: {
user: {
type: Object,
required: true,
},
},
data: function () {
return {
file: null,
tryingToCreate: false,
errors: [],
papaConfig: { dynamicTyping: true, skipEmptyLines: true, complete: this.assignResults },
fileData: null,
fileHeadings: null,
myVehicles: [],
selectedVehicle: null,
invertFullTank: null,
filledValueString: '',
notFilledValueString: '',
isFullTankString: false,
fileHeadingMap: {
fuelQuantity: null,
perUnitPrice: null,
totalAmount: null,
odoReading: null,
isTankFull: null,
hasMissedFillup: null,
comments: [], // [int]
fillingStation: null,
date: null,
fuelSubType: null,
},
}
},
computed: {
...mapState('utils', ['isMobile']),
...mapState('vehicles', ['vehicles']),
uploadButtonLabel() {
if (this.isMobile) {
if (this.file == null) {
return this.$t('choosephoto')
} else {
return ''
}
} else {
if (this.file == null) {
return this.$t('choosefile')
} else {
return ''
}
}
},
},
mounted() {
this.myVehicles = this.vehicles
},
methods: {
assignResults(results, file) {
this.fileData = results.data
this.fileHeadings = results.data[0]
},
parseCSV() {
if (this.file == null) {
return
}
this.errorMessage = ''
Papa.parse(this.file, this.papaConfig)
},
getUsedHeadings() {
return Object.keys(this.fileHeadingMap).filter((k) => this.fileHeadingMap[k] != null) // filter non-null properties
},
getTimezone() {
return Intl.DateTimeFormat().resolvedOptions().timeZone
},
csvToJson() {
const data = []
const headings = this.getUsedHeadings().reduce((a, k) => ({ ...a, [k]: this.fileHeadingMap[k] }), {}) // create new object from filter
const comments = (row) => {
return this.fileHeadingMap.comments.reduce((a, fi) => {
// TODO: sanitize to prevent XSS
return `${a}${this.fileHeadings[fi]}: ${row[fi]}\n`
}, '')
}
const calculateTotal = (row) => {
return this.fileHeadingMap.totalAmount === -1
? row[this.fileHeadingMap.fuelQuantity] * row[this.fileHeadingMap.perUnitPrice]
: row[this.fileHeadingMap.totalAmount]
}
const setFullTank = (row) => {
if (row[this.fileHeadingMap.isTankFull].toLowerCase() === this.filledValueString.toLowerCase()) {
return true
} else if (row[this.fileHeadingMap.isTankFull].toLowerCase() === this.notFilledValueString.toLowerCase()) {
return false
} else {
// TODO: need to handle errors better
throw Error
}
}
for (let r = 1; r < this.fileData.length; r++) {
const row = this.fileData[r]
const item = {}
Object.keys(headings).forEach((k) => {
if (k === 'comments') {
item[k] = comments(row)
} else if (k === 'totalAmount') {
item[k] = calculateTotal(row)
} else if (k === 'isTankFull') {
if (this.isFullTankString) {
item[k] = setFullTank(row)
} else {
if (this.invertFullTank) {
item[k] = Boolean(!row[headings[k]])
} else {
item[k] = Boolean(row[headings[k]])
}
}
} else if (k === 'hasMissedFillup') {
// TODO: need to account for this field being a string
item[k] = Boolean(row[headings[k]])
} else if (k === 'date') {
item[k] = new Date(row[headings[k]]).toISOString()
} else {
item[k] = row[headings[k]]
}
})
data.push(item)
}
return data
},
importData() {
if (this.errors.length === 0) {
try {
const content = {
data: this.csvToJson(),
vehicleId: this.selectedVehicle.id,
timezone: this.getTimezone(),
}
axios
.post('/api/import/generic', content)
.then((data) => {
this.$buefy.toast.open({
message: this.$t('importsuccessfull'),
type: 'is-success',
duration: 3000,
})
setTimeout(() => this.$router.push({ name: 'home' }), 1000)
})
.catch((ex) => {
this.$buefy.toast.open({
duration: 5000,
message: this.$t('importerror'),
position: 'is-bottom',
type: 'is-danger',
})
console.log(ex)
if (ex.response && ex.response.data.error) {
this.errors.push(ex.response.data.error)
}
})
} catch (e) {
// TODO: handle error
this.errors.push(e)
}
} else {
this.errors.push('fix errors')
}
},
checkFieldString() {
const tankFull = this.fileData[1][this.fileHeadingMap.isTankFull]
if (typeof tankFull !== 'boolean' && typeof tankFull === 'string') {
this.isFullTankString = true
}
},
clearHeadingProperty(property) {
if (property === 'comments') {
this.fileHeadingMap[property] = []
} else {
this.fileHeadingMap[property] = null
}
},
},
}
</script>
<template>
<Layout>
<div class="columns box">
<div class="column">
<h1 class="title">{{ $t('importgeneric') }}</h1>
</div>
</div>
<br />
<div v-if="fileData === null" class="columns">
<div class="column">
<p class="subtitle"> {{ $t('stepstoimport', { name: 'CSV' }) }}</p>
<ol>
<!-- <li>{{ $t('importhintcreatecsv', { 'name': 'Fuelly' }) }} <a href="http://docs.fuelly.com/acar-import-export-center" target="_nofollow">{{ $t('here') }}</a>.</li> -->
<li>{{ $t('importgenerichintdata') }}</li>
<li>{{ $t('importhintvehiclecreated') }}</li>
<li v-html="$t('importhintcurrdist')"></li>
<li v-html="$t('importhintunits')"></li>
<li>
<b>{{ $t('dontimportagain') }}</b>
</li>
</ol>
</div>
</div>
<div v-if="fileData === null" class="section box">
<div class="columns">
<div class="column is-two-thirds">
<p class="subtitle">{{ $t('choosedatafile') }}</p>
</div>
<div class="column is-one-third is-flex is-align-content-center">
<form @submit.prevent="parseCSV">
<div class="columns">
<div class="column">
<b-field class="file is-primary" :class="{ 'has-name': !!file }">
<b-upload v-model="file" class="file-label" accept=".csv" required>
<span class="file-cta">
<b-icon class="file-icon" icon="upload"></b-icon>
<span class="file-label">{{ uploadButtonLabel }}</span>
</span>
<span v-if="file" class="file-name" :class="isMobile ? 'file-name-mobile' : 'file-name-desktop'">
{{ file.name }}
</span>
</b-upload>
</b-field>
</div>
<div class="column">
<b-button tag="button" native-type="submit" type="is-primary" class="control">
{{ $t('import') }}
</b-button>
</div>
</div>
</form>
</div>
</div>
</div>
<div v-else class="columns">
<div class="column">
<p class="subtitle">Map Fields</p>
<form class="" @submit.prevent="importData">
<b-field :label="$t('selectvehicle')">
<b-select v-model="selectedVehicle" :placeholder="$t('vehicle')" required expanded>
<option v-for="option in myVehicles" :key="option.id" :value="option">
{{ option.nickname }}
</option>
</b-select>
</b-field>
<span v-if="selectedVehicle !== null">
<b-field :label="$t('fillupdate')">
<b-select v-model="fileHeadingMap.date" required expanded>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field>
<template v-slot:label>
{{ $t('fuelsubtype') }}
<b-tooltip type="is-dark" label="Clear selection">
<b-button
type="is-ghost"
size="is-small"
icon-pack="fas"
icon-right="arrow-rotate-left"
@click="clearHeadingProperty('fuelSubType')"
></b-button>
</b-tooltip>
</template>
<b-select v-model="fileHeadingMap.fuelSubType" expanded>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field :label="$t('quantity')">
<b-select v-model="fileHeadingMap.fuelQuantity" expanded required>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field :label="$t('per', { '0': $t('price'), '1': $t('unit.short.' + selectedVehicle.fuelUnitDetail.key) })">
<b-select v-model.number="fileHeadingMap.perUnitPrice" type="number" min="0" step=".001" expanded required>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field :label="$t('totalamountpaid')">
<b-select v-model.number="fileHeadingMap.totalAmount" expanded required>
<option value="-1">Calculated</option>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field :label="$t('odometer')">
<b-select v-model.number="fileHeadingMap.odoReading" expanded required>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field :label="$t('tankpartialfull')">
<b-radio-button v-model="invertFullTank" native-value="false">{{ $t('fulltank') }}</b-radio-button>
<b-radio-button v-model="invertFullTank" native-value="true">{{ $t('partialfillup') }}</b-radio-button>
</b-field>
<b-field>
<b-select v-model="fileHeadingMap.isTankFull" required @input="checkFieldString">
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<span v-if="isFullTankString === true" required>
<b-field label="Value when tank is filled">
<b-input v-model="filledValueString"></b-input>
</b-field>
<b-field label="Value when tank was not completely filled">
<b-input v-model="notFilledValueString"></b-input>
</b-field>
</span>
<b-field>
<template v-slot:label>
{{ $t('missedfillup') }}
<b-tooltip type="is-dark" label="Clear selection">
<b-button
type="is-ghost"
size="is-small"
icon-pack="fas"
icon-right="arrow-rotate-left"
@click="clearHeadingProperty('hasMissedFillup')"
></b-button>
</b-tooltip>
</template>
<b-select v-model="fileHeadingMap.hasMissedFillup">
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field>
<template v-slot:label>
{{ $t('fillingstation') }}
<b-tooltip type="is-dark" label="Clear selection">
<b-button
type="is-ghost"
size="is-small"
icon-pack="fas"
icon-right="arrow-rotate-left"
@click="clearHeadingProperty('fillingStation')"
></b-button>
</b-tooltip>
</template>
<b-select v-model="fileHeadingMap.fillingStation">
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<b-field>
<template v-slot:label>
{{ $t('comments') }}
<b-tooltip type="is-dark" label="Clear selection">
<b-button
type="is-ghost"
size="is-small"
icon-pack="fas"
icon-right="arrow-rotate-left"
@click="clearHeadingProperty('comments')"
></b-button>
</b-tooltip>
</template>
<b-select v-model="fileHeadingMap.comments" type="textarea" multiple expanded>
<option v-for="(option, index) in fileHeadings" :key="index" :value="index">
{{ option }}
</option>
</b-select>
</b-field>
<br />
<b-field>
<b-button tag="button" native-type="submit" type="is-primary" :value="$t('save')" :label="$t('import')" expanded />
<p v-if="authError"> There was an error logging in to your account. </p>
</b-field>
</span>
</form>
</div>
</div>
<b-message v-if="errors.length" type="is-danger">
<ul>
<li v-for="error in errors" :key="error">{{ error }}</li>
</ul>
</b-message>
</Layout>
</template>

View File

@@ -18,18 +18,19 @@ export default {
<template>
<Layout>
<div class="columns box"
><div class="column">
<div class="columns box">
<div class="column">
<h1 class="title">{{ $t('importdata') }}</h1>
<p class="subtitle">{{ $t('importdatadesc') }}</p>
</div></div
>
</div>
</div>
<br />
<div class="columns">
<div class="column is-one-third">
<div class="box">
<h1 class="title">Fuelly</h1>
<p>If you have been using Fuelly to store your vehicle data, export the CSV file from Fuelly and click here to import.</p>
<p>If you have been using Fuelly to store your vehicle data, export the CSV file from Fuelly and click here to
import.</p>
<br />
<b-button type="is-primary" tag="router-link" to="/import/fuelly">{{ $t('import') }}</b-button>
</div>
@@ -43,6 +44,16 @@ export default {
<b-button type="is-primary" tag="router-link" to="/import/drivvo">{{ $t('import') }}</b-button>
</div>
</div>
<div class="column is-one-third" to="/import-generic">
<div class="box">
<h1 class="title">{{ $t('importgeneric') }}</h1>
<p>{{ $t('importgenericdesc') }}</p>
<br />
<b-button type="is-primary" tag="router-link" to="/import/generic">{{ $t('import') }}</b-button>
</div>
</div>
</div>
</Layout>
</template>