simplify process. merge step 1 and 2 to one single command

.env.example

@@ -47,3 +47,6 @@ PUSHER_APP_CLUSTER=mt1
 
 MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
 MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
+
+ECOWITT_ACCOUNT=
+ECOWITT_PASSWORD=

.gitignore

@@ -3,6 +3,7 @@
 /public/storage
 /storage/*.key
 /vendor
+.idea
 .env
 .env.backup
 .phpunit.result.cache

README.md

@@ -4,108 +4,31 @@
 
 ## General
 
-In case your weather station send all data to ecowitt.net and you lost your weewx database, this huge cannonball will help you to reintegrate your data.
+In case your weather station send all data to ecowitt.net and you lost your weewx database,
+this huge cannonball will help you to reintegrate your data.
 
 The steps are easy.
 
-### 1. Fetch
+### 1. Convert
 
-Get your data from 'undocumentend' ecowitt API.
+This simple tool will login into your ecowitt.net account, fetch all available devices and download all available data
-
+for the range between `startdate` and `enddate`.
-Use browser inspect tools to get session_id after login. You find this information within your cookies OR you can inspect XHR request.
-
-Replace SESSION_ID and DEVICE_ID in this bash script.
-
-*HINT*: I'm running Mac OS. You need to install `coreutils` before. **=>** `brew install coreutils`
-
-This dirty script can download your data on a daily base or per week. Adjust the script to your needs.
-It will create JSON Files in your current directory. These JSON Files have to be converted in step 2.
 
 ```sh
-#!/bin/bash
+php artisan ecowitt:export {startdate} {enddate}
-SESSION_ID="<MY SESSION ID>" #
-DEVICE_ID="<MY DEVICE ID>" # Fetch this ID from ecowitt URL. https://www.ecowitt.net/home/index?id=<MYID>
-
-function week2date () {
-  local year=$1
-  local week=$2
-  local dayofweek=$3
-  gdate -d "$year-01-01 +$(( $week * 7 + 1 - $(gdate -d "$year-01-04" +%u ) - 3 )) days -2 days + $dayofweek days" +"%Y-%m-%d"
-}
-
-function getDateOfDay() {
-	local day=$1
-	local year=$2
-	gdate -d "$year-1-1 +$day days" +%F
-}
-
-function loadByDate() {
-	local date=$1
-	curl -s 'https://webapi.www.ecowitt.net/index/get_data' \
-		-X 'POST' \
-		-H 'Accept: application/json, text/plain, */*' \
-		-H 'Content-Type: application/x-www-form-urlencoded' \
-		-H 'Origin: https://www.ecowitt.net' \
-		-H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \
-		-H 'Content-Length: 93' \
-		-H 'Accept-Language: en-us' \
-		-H 'Host: webapi.www.ecowitt.net' \
-		-H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \
-		-H 'Accept-Encoding: gzip, deflate, br' \
-		-H 'Connection: keep-alive' \
-		--data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${date}%2000%3A00&edate=${date}%2023%3A59&page=1" | gunzip > $date.json
-}
-
-
-function loadByWeek() {
-	local start=$1;
-	local end=$2;
-	local week=$3;
-	curl 'https://webapi.www.ecowitt.net/index/get_data' \
-		-X 'POST' \
-		-H 'Accept: application/json, text/plain, */*' \
-		-H 'Content-Type: application/x-www-form-urlencoded' \
-		-H 'Origin: https://www.ecowitt.net' \
-		-H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \
-		-H 'Content-Length: 93' \
-		-H 'Accept-Language: en-us' \
-		-H 'Host: webapi.www.ecowitt.net' \
-		-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15' \
-		-H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \
-		-H 'Accept-Encoding: gzip, deflate, br' \
-		-H 'Connection: keep-alive' \
-		--data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${start}%2001%3A00&edate=${end}%2023%3A59&page=1" | gunzip > $week.json
-}
-
-for day in {0..365}; do
-	DATE=$(getDateOfDay $day 2020);
-	echo "Fetching Date: $DATE";
-	loadByDate $DATE;
-	sleep 1;
-done
-
-for week in {1..53}; do
-	STARTDATE=$(week2date 2020 $week 1);
-	ENDDATE=$(week2date 2020 $week 7);
-	echo "Fetching Week $week ..."
-
-	loadByWeek $STARTDATE $ENDDATE $week
-done
 ```
 
-### 2. Convert
+example
-
-Convert json files to importable format for wee_import using this cannonball tool:
-
 ```sh
-php artisan ecowitt:convert {path of json files}
+php artisan ecowitt:export 2020-01-01 2020-12-31
 ```
 
 Details see source file: `/app/Console/Commands/EcowittConvertData.php`
 
-This script will generate a CSV File which is compatible with wee_import config in step 3. The file will be generated in `/storage/app/weewx.csv`.
+This script will generate a CSV File which is compatible with wee_import config in step 2.
+The file will be generated in `/storage/app/ecowitt_<device_id>.csv`.
 
-### 3. import
+### 2. import
 
 Import your data with wee_import. Sample config below
 
@@ -309,4 +232,4 @@ source = CSV
 
 **DONE!**
 
-regenerate / restart weewx daemon and check your data!
+regenerate / restart weewx daemon and check your data!

app/Console/Commands/EcowittConvertData.php

@@ -3,19 +3,27 @@
 namespace App\Console\Commands;
 
 use App\Exports\WeewxExport;
+use Carbon\Carbon;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Http;
 use Maatwebsite\Excel\Facades\Excel;
 use SplFileInfo;
 
 class EcowittConvertData extends Command
 {
+    /** @var Carbon */
+    protected $startDate;
+
+    /** @var Carbon */
+    protected $endDate;
+
     /**
      * The name and signature of the console command.
      *
      * @var string
      */
-    protected $signature = 'ecowitt:convert {path}';
+    protected $signature = 'ecowitt:export {startDate} {endDate}';
 
     /**
      * The console command description.
@@ -41,19 +49,42 @@ class EcowittConvertData extends Command
      */
     public function handle()
     {
-        $path = $this->argument('path');
+        $session_id = $this->getSessionId();
-        $files = collect(File::allFiles($path));
+        $device_ids = $this->getDeviceIds($session_id);
-        $outputData = [];
+
-        $files->each(function(SplFileInfo $file) use (&$outputData) {
+        $this->startDate = Carbon::parse($this->argument('startDate'))->startOfDay()
-            $ecowitt = json_decode(File::get($file->getPathname()),true);
+            ->format('Y-m-d H:i');
-            if (empty(data_get($ecowitt, 'list.tempf'))) {
+
-                $this->comment("skip '{$file->getBasename()}'. next file");
+        $this->startDate = Carbon::parse($this->argument('endDate'))->endOfDay()
-                return;
+            ->format('Y-m-d H:i');
-            }
+
+        $device_ids->each(function ($deviceId) use ($session_id) {
+
+            $startDate = $this->startDate;
+            $endDate = $this->endDate;
+
+            $response = Http::withCookies(
+                [
+                    'ousaite_session' => $session_id,
+                ], 'www.ecowitt.net')
+                ->asForm()
+                ->post('https://webapi.www.ecowitt.net/index/get_data', [
+                    'device_id' => $deviceId,
+                    'is_list' => 0,
+                    'mode' => 0,
+                    'sdate' => $startDate,
+                    'edate' => $endDate,
+                    'page' => 1,
+                ]);
+
+            $ecowitt = $response->json();
+
+            // declare output variable
+            $outputData = [];
 
             // Temperature in C
             $outdoorTemp = $this->getData($ecowitt, 'list.tempf.list.tempf');
-            
+
             // Feels Like in C
             $outdoorTempGust = $this->getData($ecowitt, 'list.tempf.list.sendible_temp');
 
@@ -74,7 +105,7 @@ class EcowittConvertData extends Command
 
             // uv
             $uvi = $this->getData($ecowitt, 'list.uv.list.uv');
-            
+
             // rainrate in mm/hr b
             $rainRateH = $this->getData($ecowitt, 'list.rain.list.rainratein');
 
@@ -96,7 +127,7 @@ class EcowittConvertData extends Command
             // pressure absolute in hPa
             $pressureAbs = $this->getData($ecowitt, 'list.pressure.list.baromabsin');
 
-            foreach($outdoorTemp as $date => $temp) {
+            foreach ($outdoorTemp as $date => $temp) {
                 $tmp = [
                     'date_and_time' => $date,                               // %Y-%m-%d %H:%M:%S
                     'temp_out' => $temp,                                    // degree
@@ -121,14 +152,65 @@ class EcowittConvertData extends Command
                 ];
                 $outputData[] = $tmp;
             }
+
+            Excel::store(
+                new WeewxExport($outputData),
+                "ecowitt_{$deviceId}.csv",
+                null,
+                \Maatwebsite\Excel\Excel::CSV
+            );
         });
-        Excel::store(new WeewxExport($outputData), 'weewx.csv', null, \Maatwebsite\Excel\Excel::CSV);
     }
 
-    public function getData($stack, $key) {
+    protected function getData($stack, $key)
+    {
         return collect(data_get($stack, $key))
             ->mapWithKeys(function ($value) {
                 return [$value[0] => $value[1] ?: null];
             });
     }
+
+    /**
+     * simulate login to ecowitt.net and store session_id from cookie
+     *
+     * @return mixed
+     */
+    protected function getSessionId()
+    {
+        $response = Http::asForm()->post('https://webapi.www.ecowitt.net/user/site/login', [
+            'account' => env('ECOWITT_ACCOUNT'),
+            'password' => env('ECOWITT_PASSWORD'),
+            'authorize' => '',
+        ]);
+
+        $loginData = $response->json();
+
+        $cookies = $response->cookies();
+
+        return $cookies->getCookieByName('ousaite_session')->getValue();
+    }
+
+    /**
+     * fetch all available device IDs
+     * @param $session_id
+     * @return \Illuminate\Support\Collection
+     */
+    protected function getDeviceIds($session_id): \Illuminate\Support\Collection
+    {
+        $deviceResponse = Http::withCookies(
+            [
+                'ousaite_session' => $session_id,
+            ], 'www.ecowitt.net')
+            ->asForm()
+            ->post('https://webapi.www.ecowitt.net/index/get_devices', [
+                'uid' => '',
+                'type' => 1
+            ]);
+
+        $devices = collect(data_get($deviceResponse->json(), 'device_list'));
+
+        return $devices->map(function ($device) {
+            return data_get($device, 'id');
+        });
+    }
 }

composer.json

@@ -11,7 +11,7 @@
         "php": "^7.3|^8.0",
         "fideloper/proxy": "^4.4",
         "fruitcake/laravel-cors": "^2.0",
-        "guzzlehttp/guzzle": "^7.0.1",
+        "guzzlehttp/guzzle": "^7.2",
         "laravel/framework": "^8.12",
         "laravel/tinker": "^2.5",
         "maatwebsite/excel": "^3.1"

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ad0023f01098d001a84f01dcfc5def40",
+    "content-hash": "36fa813c4a4a2cd2fcc15dfde8f14463",
     "packages": [
         {
             "name": "asm89/stack-cors",

routes/console.php

@@ -4,7 +4,7 @@ use Illuminate\Foundation\Inspiring;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\File;
-
+use Illuminate\Support\Facades\Http;
 /*
 |--------------------------------------------------------------------------
 | Console Routes