simplify process. merge step 1 and 2 to one single command

2020-12-29 22:07:39 +01:00
parent dbf4fa13c0
commit 102cdb51ea
7 changed files with 116 additions and 107 deletions
--- a/.env.example
+++ b/.env.example
@@ -47,3 +47,6 @@ PUSHER_APP_CLUSTER=mt1

 MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
 MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
+
+ECOWITT_ACCOUNT=
+ECOWITT_PASSWORD=
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 /public/storage
 /storage/*.key
 /vendor
+.idea
 .env
 .env.backup
 .phpunit.result.cache
--- a/README.md
+++ b/README.md
@@ -4,108 +4,31 @@

 ## General

-In case your weather station send all data to ecowitt.net and you lost your weewx database, this huge cannonball will help you to reintegrate your data.
+In case your weather station send all data to ecowitt.net and you lost your weewx database,
+this huge cannonball will help you to reintegrate your data.

 The steps are easy.

-### 1. Fetch
+### 1. Convert

-Get your data from 'undocumentend' ecowitt API.
-
-Use browser inspect tools to get session_id after login. You find this information within your cookies OR you can inspect XHR request.
-
-Replace SESSION_ID and DEVICE_ID in this bash script.
-
-*HINT*: I'm running Mac OS. You need to install `coreutils` before. **=>** `brew install coreutils`
-
-This dirty script can download your data on a daily base or per week. Adjust the script to your needs.
-It will create JSON Files in your current directory. These JSON Files have to be converted in step 2.
+This simple tool will login into your ecowitt.net account, fetch all available devices and download all available data
+for the range between `startdate` and `enddate`.

 ```sh
-#!/bin/bash
-SESSION_ID="<MY SESSION ID>" #
-DEVICE_ID="<MY DEVICE ID>" # Fetch this ID from ecowitt URL. https://www.ecowitt.net/home/index?id=<MYID>
-
-function week2date () {
-  local year=$1
-  local week=$2
-  local dayofweek=$3
-  gdate -d "$year-01-01 +$(( $week * 7 + 1 - $(gdate -d "$year-01-04" +%u ) - 3 )) days -2 days + $dayofweek days" +"%Y-%m-%d"
-}
-
-function getDateOfDay() {
-	local day=$1
-	local year=$2
-	gdate -d "$year-1-1 +$day days" +%F
-}
-
-function loadByDate() {
-	local date=$1
-	curl -s 'https://webapi.www.ecowitt.net/index/get_data' \
-		-X 'POST' \
-		-H 'Accept: application/json, text/plain, */*' \
-		-H 'Content-Type: application/x-www-form-urlencoded' \
-		-H 'Origin: https://www.ecowitt.net' \
-		-H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \
-		-H 'Content-Length: 93' \
-		-H 'Accept-Language: en-us' \
-		-H 'Host: webapi.www.ecowitt.net' \
-		-H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \
-		-H 'Accept-Encoding: gzip, deflate, br' \
-		-H 'Connection: keep-alive' \
-		--data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${date}%2000%3A00&edate=${date}%2023%3A59&page=1" | gunzip > $date.json
-}
-
-
-function loadByWeek() {
-	local start=$1;
-	local end=$2;
-	local week=$3;
-	curl 'https://webapi.www.ecowitt.net/index/get_data' \
-		-X 'POST' \
-		-H 'Accept: application/json, text/plain, */*' \
-		-H 'Content-Type: application/x-www-form-urlencoded' \
-		-H 'Origin: https://www.ecowitt.net' \
-		-H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \
-		-H 'Content-Length: 93' \
-		-H 'Accept-Language: en-us' \
-		-H 'Host: webapi.www.ecowitt.net' \
-		-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15' \
-		-H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \
-		-H 'Accept-Encoding: gzip, deflate, br' \
-		-H 'Connection: keep-alive' \
-		--data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${start}%2001%3A00&edate=${end}%2023%3A59&page=1" | gunzip > $week.json
-}
-
-for day in {0..365}; do
-	DATE=$(getDateOfDay $day 2020);
-	echo "Fetching Date: $DATE";
-	loadByDate $DATE;
-	sleep 1;
-done
-
-for week in {1..53}; do
-	STARTDATE=$(week2date 2020 $week 1);
-	ENDDATE=$(week2date 2020 $week 7);
-	echo "Fetching Week $week ..."
-
-	loadByWeek $STARTDATE $ENDDATE $week
-done
+php artisan ecowitt:export {startdate} {enddate}
 ```

-### 2. Convert
-
-Convert json files to importable format for wee_import using this cannonball tool:
-
+example
 ```sh
-php artisan ecowitt:convert {path of json files}
+php artisan ecowitt:export 2020-01-01 2020-12-31
 ```

 Details see source file: `/app/Console/Commands/EcowittConvertData.php`

-This script will generate a CSV File which is compatible with wee_import config in step 3. The file will be generated in `/storage/app/weewx.csv`.
+This script will generate a CSV File which is compatible with wee_import config in step 2.
+The file will be generated in `/storage/app/ecowitt_<device_id>.csv`.

-### 3. import
+### 2. import

 Import your data with wee_import. Sample config below

@@ -309,4 +232,4 @@ source = CSV

 **DONE!**

-regenerate / restart weewx daemon and check your data!
+regenerate / restart weewx daemon and check your data!
--- a/app/Console/Commands/EcowittConvertData.php
+++ b/app/Console/Commands/EcowittConvertData.php
@@ -3,19 +3,27 @@
 namespace App\Console\Commands;

 use App\Exports\WeewxExport;
+use Carbon\Carbon;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Http;
 use Maatwebsite\Excel\Facades\Excel;
 use SplFileInfo;

 class EcowittConvertData extends Command
 {
+    /** @var Carbon */
+    protected $startDate;
+
+    /** @var Carbon */
+    protected $endDate;
+
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
-    protected $signature = 'ecowitt:convert {path}';
+    protected $signature = 'ecowitt:export {startDate} {endDate}';

    /**
     * The console command description.
@@ -41,19 +49,42 @@ class EcowittConvertData extends Command
     */
    public function handle()
    {
-        $path = $this->argument('path');
-        $files = collect(File::allFiles($path));
-        $outputData = [];
-        $files->each(function(SplFileInfo $file) use (&$outputData) {
-            $ecowitt = json_decode(File::get($file->getPathname()),true);
-            if (empty(data_get($ecowitt, 'list.tempf'))) {
-                $this->comment("skip '{$file->getBasename()}'. next file");
-                return;
-            }
+        $session_id = $this->getSessionId();
+        $device_ids = $this->getDeviceIds($session_id);
+
+        $this->startDate = Carbon::parse($this->argument('startDate'))->startOfDay()
+            ->format('Y-m-d H:i');
+
+        $this->startDate = Carbon::parse($this->argument('endDate'))->endOfDay()
+            ->format('Y-m-d H:i');
+
+        $device_ids->each(function ($deviceId) use ($session_id) {
+
+            $startDate = $this->startDate;
+            $endDate = $this->endDate;
+
+            $response = Http::withCookies(
+                [
+                    'ousaite_session' => $session_id,
+                ], 'www.ecowitt.net')
+                ->asForm()
+                ->post('https://webapi.www.ecowitt.net/index/get_data', [
+                    'device_id' => $deviceId,
+                    'is_list' => 0,
+                    'mode' => 0,
+                    'sdate' => $startDate,
+                    'edate' => $endDate,
+                    'page' => 1,
+                ]);
+
+            $ecowitt = $response->json();
+
+            // declare output variable
+            $outputData = [];

            // Temperature in C
            $outdoorTemp = $this->getData($ecowitt, 'list.tempf.list.tempf');
-            
+
            // Feels Like in C
            $outdoorTempGust = $this->getData($ecowitt, 'list.tempf.list.sendible_temp');

@@ -74,7 +105,7 @@ class EcowittConvertData extends Command

            // uv
            $uvi = $this->getData($ecowitt, 'list.uv.list.uv');
-            
+
            // rainrate in mm/hr b
            $rainRateH = $this->getData($ecowitt, 'list.rain.list.rainratein');

@@ -96,7 +127,7 @@ class EcowittConvertData extends Command
            // pressure absolute in hPa
            $pressureAbs = $this->getData($ecowitt, 'list.pressure.list.baromabsin');

-            foreach($outdoorTemp as $date => $temp) {
+            foreach ($outdoorTemp as $date => $temp) {
                $tmp = [
                    'date_and_time' => $date,                               // %Y-%m-%d %H:%M:%S
                    'temp_out' => $temp,                                    // degree
@@ -121,14 +152,65 @@ class EcowittConvertData extends Command
                ];
                $outputData[] = $tmp;
            }
+
+            Excel::store(
+                new WeewxExport($outputData),
+                "ecowitt_{$deviceId}.csv",
+                null,
+                \Maatwebsite\Excel\Excel::CSV
+            );
        });
-        Excel::store(new WeewxExport($outputData), 'weewx.csv', null, \Maatwebsite\Excel\Excel::CSV);
    }

-    public function getData($stack, $key) {
+    protected function getData($stack, $key)
+    {
        return collect(data_get($stack, $key))
            ->mapWithKeys(function ($value) {
                return [$value[0] => $value[1] ?: null];
            });
    }
+
+    /**
+     * simulate login to ecowitt.net and store session_id from cookie
+     *
+     * @return mixed
+     */
+    protected function getSessionId()
+    {
+        $response = Http::asForm()->post('https://webapi.www.ecowitt.net/user/site/login', [
+            'account' => env('ECOWITT_ACCOUNT'),
+            'password' => env('ECOWITT_PASSWORD'),
+            'authorize' => '',
+        ]);
+
+        $loginData = $response->json();
+
+        $cookies = $response->cookies();
+
+        return $cookies->getCookieByName('ousaite_session')->getValue();
+    }
+
+    /**
+     * fetch all available device IDs
+     * @param $session_id
+     * @return \Illuminate\Support\Collection
+     */
+    protected function getDeviceIds($session_id): \Illuminate\Support\Collection
+    {
+        $deviceResponse = Http::withCookies(
+            [
+                'ousaite_session' => $session_id,
+            ], 'www.ecowitt.net')
+            ->asForm()
+            ->post('https://webapi.www.ecowitt.net/index/get_devices', [
+                'uid' => '',
+                'type' => 1
+            ]);
+
+        $devices = collect(data_get($deviceResponse->json(), 'device_list'));
+
+        return $devices->map(function ($device) {
+            return data_get($device, 'id');
+        });
+    }
 }
--- a/composer.json
+++ b/composer.json
@@ -11,7 +11,7 @@
        "php": "^7.3|^8.0",
        "fideloper/proxy": "^4.4",
        "fruitcake/laravel-cors": "^2.0",
-        "guzzlehttp/guzzle": "^7.0.1",
+        "guzzlehttp/guzzle": "^7.2",
        "laravel/framework": "^8.12",
        "laravel/tinker": "^2.5",
        "maatwebsite/excel": "^3.1"
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "ad0023f01098d001a84f01dcfc5def40",
+    "content-hash": "36fa813c4a4a2cd2fcc15dfde8f14463",
    "packages": [
        {
            "name": "asm89/stack-cors",
--- a/routes/console.php
+++ b/routes/console.php
@@ -4,7 +4,7 @@ use Illuminate\Foundation\Inspiring;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\File;
-
+use Illuminate\Support\Facades\Http;
 /*
 |--------------------------------------------------------------------------
 | Console Routes