From 11960bc6665cd38f7b869a587996d20eeafa316c Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 7 Dec 2017 16:33:54 +0100 Subject: [PATCH 01/25] Added status handler --- handlers_stats.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 1 + 2 files changed, 70 insertions(+) create mode 100644 handlers_stats.go diff --git a/handlers_stats.go b/handlers_stats.go new file mode 100644 index 00000000..eb5e0b5f --- /dev/null +++ b/handlers_stats.go @@ -0,0 +1,69 @@ +// +// This file is part of arduino-connector +// +// Copyright (C) 2017 Arduino AG (http://www.arduino.cc/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/arduino/go-system-stats/disk" + "github.com/arduino/go-system-stats/mem" + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// StatsCB sends statistics about resource used in the system (RAM, Disk, Network, etc...) +func StatsCB(s *Status) mqtt.MessageHandler { + return func(client mqtt.Client, msg mqtt.Message) { + // Gather all system data metrics + memStats, err := mem.GetStats() + if err != nil { + s.Error("/stats/error", fmt.Errorf("Retrieving memory stats: %s", err)) + return + } + + diskStats, err := disk.GetStats() + if err != nil { + s.Error("/stats/error", fmt.Errorf("Retrieving disk stats: %s", err)) + return + } + + type StatsPayload struct { + Memory *mem.Stats `json:"memory"` + Disk []*disk.FSStats `json:"disk"` + } + + info := StatsPayload{ + Memory: memStats, + Disk: diskStats, + } + + // Send result + data, err := json.Marshal(info) + if err != nil { + s.Error("/stats/error", fmt.Errorf("Json marsahl result: %s", err)) + return + } + + //var out bytes.Buffer + //json.Indent(&out, data, "", " ") + //fmt.Println(string(out.Bytes())) + + s.Info("/stats", string(data)+"\n") + } +} diff --git a/main.go b/main.go index fc36c907..a5b870c7 100644 --- a/main.go +++ b/main.go @@ -211,6 +211,7 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { mqttClient.Subscribe("$aws/things/"+id+"/upload/post", 1, UploadCB(status)) mqttClient.Subscribe("$aws/things/"+id+"/sketch/post", 1, SketchCB(status)) mqttClient.Subscribe("$aws/things/"+id+"/update/post", 1, UpdateCB(status)) + mqttClient.Subscribe("$aws/things/"+id+"/stats/post", 1, StatsCB(status)) } func addFileToSketchDB(file os.FileInfo, status *Status) *SketchStatus { From d324c03b19c9200ba16d083d458c58cbad842f93 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 13 Dec 2017 11:33:40 +0100 Subject: [PATCH 02/25] Removed one level of indirection in mqtt callbacks --- handlers.go | 266 ++++++++++++++++++++++------------------------ handlers_stats.go | 64 ++++++----- main.go | 10 +- 3 files changed, 165 insertions(+), 175 deletions(-) diff --git a/handlers.go b/handlers.go index 5ca95259..675f0e1e 100644 --- a/handlers.go +++ b/handlers.go @@ -42,11 +42,9 @@ import ( "golang.org/x/crypto/ssh/terminal" ) -// StatusCB replies with the current status of the arduino-connector -func StatusCB(status *Status) mqtt.MessageHandler { - return func(client mqtt.Client, msg mqtt.Message) { - status.Publish() - } +// StatusEvent replies with the current status of the arduino-connector +func (status *Status) StatusEvent(client mqtt.Client, msg mqtt.Message) { + status.Publish() } type UpdatePayload struct { @@ -55,50 +53,48 @@ type UpdatePayload struct { Token string `json:"token"` } -// UpdateCB handles the connector autoupdate +// UpdateEvent handles the connector autoupdate // Any URL must be signed with Arduino private key -func UpdateCB(status *Status) mqtt.MessageHandler { - return func(client mqtt.Client, msg mqtt.Message) { - var info UpdatePayload - err := json.Unmarshal(msg.Payload(), &info) - if err != nil { - status.Error("/update", errors.Wrapf(err, "unmarshal %s", msg.Payload())) - return - } - executablePath, _ := os.Executable() - name := filepath.Join(os.TempDir(), filepath.Base(executablePath)) - err = downloadFile(name, info.URL, info.Token) - err = downloadFile(name+".sig", info.URL+".sig", info.Token) - if err != nil { - status.Error("/update", errors.Wrap(err, "no signature file "+info.URL+".sig")) - return - } - // check the signature - err = checkGPGSig(name, name+".sig") - if err != nil { - status.Error("/update", errors.Wrap(err, "wrong signature "+info.URL+".sig")) - return - } - // chmod it - err = os.Chmod(name, 0744) - if err != nil { - status.Error("/update", errors.Wrapf(err, "chmod 744 %s", name)) - return - } - os.Rename(executablePath, executablePath+".old") - // copy it over existing binary - err = copyFileAndRemoveOriginal(name, executablePath) - if err != nil { - // rollback - os.Rename(executablePath+".old", executablePath) - status.Error("/update", errors.Wrap(err, "error copying itself from "+name+" to "+executablePath)) - return - } - os.Chmod(executablePath, 0744) - os.Remove(executablePath + ".old") - // leap of faith: kill itself, systemd should respawn the process - os.Exit(0) +func (status *Status) UpdateEvent(client mqtt.Client, msg mqtt.Message) { + var info UpdatePayload + err := json.Unmarshal(msg.Payload(), &info) + if err != nil { + status.Error("/update", errors.Wrapf(err, "unmarshal %s", msg.Payload())) + return + } + executablePath, _ := os.Executable() + name := filepath.Join(os.TempDir(), filepath.Base(executablePath)) + err = downloadFile(name, info.URL, info.Token) + err = downloadFile(name+".sig", info.URL+".sig", info.Token) + if err != nil { + status.Error("/update", errors.Wrap(err, "no signature file "+info.URL+".sig")) + return + } + // check the signature + err = checkGPGSig(name, name+".sig") + if err != nil { + status.Error("/update", errors.Wrap(err, "wrong signature "+info.URL+".sig")) + return + } + // chmod it + err = os.Chmod(name, 0744) + if err != nil { + status.Error("/update", errors.Wrapf(err, "chmod 744 %s", name)) + return + } + os.Rename(executablePath, executablePath+".old") + // copy it over existing binary + err = copyFileAndRemoveOriginal(name, executablePath) + if err != nil { + // rollback + os.Rename(executablePath+".old", executablePath) + status.Error("/update", errors.Wrap(err, "error copying itself from "+name+" to "+executablePath)) + return } + os.Chmod(executablePath, 0744) + os.Remove(executablePath + ".old") + // leap of faith: kill itself, systemd should respawn the process + os.Exit(0) } // UploadPayload contains the name and url of the sketch to upload on the device @@ -109,91 +105,89 @@ type UploadPayload struct { Token string `json:"token"` } -// UploadCB receives the url and name of the sketch binary, then it +// UploadEvent receives the url and name of the sketch binary, then it // - downloads the binary, // - chmods +x it // - executes redirecting stdout and sterr to a proper logger -func UploadCB(status *Status) mqtt.MessageHandler { - return func(client mqtt.Client, msg mqtt.Message) { - // unmarshal - var info UploadPayload - var sketch SketchStatus - err := json.Unmarshal(msg.Payload(), &info) - if err != nil { - status.Error("/upload", errors.Wrapf(err, "unmarshal %s", msg.Payload())) - return - } - - if info.ID == "" { - info.ID = info.Name - } - - // Stop and delete if existing - if sketch, ok := status.Sketches[info.ID]; ok { - err = applyAction(sketch, "STOP", status) - if err != nil { - status.Error("/upload", errors.Wrapf(err, "stop pid %d", sketch.PID)) - return - } +func (status *Status) UploadEvent(client mqtt.Client, msg mqtt.Message) { + // unmarshal + var info UploadPayload + var sketch SketchStatus + err := json.Unmarshal(msg.Payload(), &info) + if err != nil { + status.Error("/upload", errors.Wrapf(err, "unmarshal %s", msg.Payload())) + return + } - sketchFolder, err := GetSketchFolder() - err = os.Remove(filepath.Join(sketchFolder, sketch.Name)) - if err != nil { - status.Error("/upload", errors.Wrapf(err, "remove %d", sketch.Name)) - return - } - } + if info.ID == "" { + info.ID = info.Name + } - folder, err := GetSketchFolder() + // Stop and delete if existing + if sketch, ok := status.Sketches[info.ID]; ok { + err = applyAction(sketch, "STOP", status) if err != nil { - status.Error("/upload", errors.Wrapf(err, "create sketch folder %s", info.ID)) + status.Error("/upload", errors.Wrapf(err, "stop pid %d", sketch.PID)) return } - // download the binary - name := filepath.Join(folder, info.Name) - err = downloadFile(name, info.URL, info.Token) + sketchFolder, err := GetSketchFolder() + err = os.Remove(filepath.Join(sketchFolder, sketch.Name)) if err != nil { - status.Error("/upload", errors.Wrapf(err, "download file %s", info.URL)) + status.Error("/upload", errors.Wrapf(err, "remove %d", sketch.Name)) return } + } - // chmod it - err = os.Chmod(name, 0744) - if err != nil { - status.Error("/upload", errors.Wrapf(err, "chmod 744 %s", name)) - return - } + folder, err := GetSketchFolder() + if err != nil { + status.Error("/upload", errors.Wrapf(err, "create sketch folder %s", info.ID)) + return + } - sketch.ID = info.ID - sketch.Name = info.Name - // save ID-Name to a sort of DB - InsertSketchInDB(sketch.Name, sketch.ID) + // download the binary + name := filepath.Join(folder, info.Name) + err = downloadFile(name, info.URL, info.Token) + if err != nil { + status.Error("/upload", errors.Wrapf(err, "download file %s", info.URL)) + return + } - // spawn process - pid, _, _, err := spawnProcess(name, &sketch, status) - if err != nil { - status.Error("/upload", errors.Wrapf(err, "spawn %s", name)) - return - } + // chmod it + err = os.Chmod(name, 0744) + if err != nil { + status.Error("/upload", errors.Wrapf(err, "chmod 744 %s", name)) + return + } - status.Info("/upload", "Sketch started with PID "+strconv.Itoa(pid)) + sketch.ID = info.ID + sketch.Name = info.Name + // save ID-Name to a sort of DB + InsertSketchInDB(sketch.Name, sketch.ID) - sketch.PID = pid - sketch.Status = "RUNNING" + // spawn process + pid, _, _, err := spawnProcess(name, &sketch, status) + if err != nil { + status.Error("/upload", errors.Wrapf(err, "spawn %s", name)) + return + } - status.Set(info.ID, &sketch) - status.Publish() + status.Info("/upload", "Sketch started with PID "+strconv.Itoa(pid)) - // go func(stdout io.ReadCloser) { - // in := bufio.NewScanner(stdout) - // for { - // for in.Scan() { - // fmt.Printf(in.Text()) // write each line to your log, or anything you need - // } - // } - // }(stdout) - } + sketch.PID = pid + sketch.Status = "RUNNING" + + status.Set(info.ID, &sketch) + status.Publish() + + // go func(stdout io.ReadCloser) { + // in := bufio.NewScanner(stdout) + // for { + // for in.Scan() { + // fmt.Printf(in.Text()) // write each line to your log, or anything you need + // } + // } + // }(stdout) } func GetSketchFolder() (string, error) { @@ -272,36 +266,34 @@ type SketchActionPayload struct { Action string } -// SketchCB listens to commands to start and stop sketches -func SketchCB(status *Status) mqtt.MessageHandler { - return func(client mqtt.Client, msg mqtt.Message) { - // unmarshal - var info SketchActionPayload - err := json.Unmarshal(msg.Payload(), &info) - if err != nil { - status.Error("/sketch", errors.Wrapf(err, "unmarshal %s", msg.Payload())) - return - } - - if info.ID == "" { - info.ID = info.Name - } +// SketchEvent listens to commands to start and stop sketches +func (status *Status) SketchEvent(client mqtt.Client, msg mqtt.Message) { + // unmarshal + var info SketchActionPayload + err := json.Unmarshal(msg.Payload(), &info) + if err != nil { + status.Error("/sketch", errors.Wrapf(err, "unmarshal %s", msg.Payload())) + return + } - if sketch, ok := status.Sketches[info.ID]; ok { - err := applyAction(sketch, info.Action, status) - if err != nil { - status.Error("/sketch", errors.Wrapf(err, "applying %s to %s", info.Action, info.Name)) - return - } - status.Info("/sketch", "successfully performed "+info.Action+" on sketch "+info.ID) + if info.ID == "" { + info.ID = info.Name + } - status.Set(info.ID, sketch) - status.Publish() + if sketch, ok := status.Sketches[info.ID]; ok { + err := applyAction(sketch, info.Action, status) + if err != nil { + status.Error("/sketch", errors.Wrapf(err, "applying %s to %s", info.Action, info.Name)) return } + status.Info("/sketch", "successfully performed "+info.Action+" on sketch "+info.ID) - status.Error("/sketch", errors.New("sketch "+info.ID+" not found")) + status.Set(info.ID, sketch) + status.Publish() + return } + + status.Error("/sketch", errors.New("sketch "+info.ID+" not found")) } func NatsCloudCB(s *Status) nats.MsgHandler { diff --git a/handlers_stats.go b/handlers_stats.go index eb5e0b5f..b213dfb0 100644 --- a/handlers_stats.go +++ b/handlers_stats.go @@ -27,43 +27,41 @@ import ( mqtt "github.com/eclipse/paho.mqtt.golang" ) -// StatsCB sends statistics about resource used in the system (RAM, Disk, Network, etc...) -func StatsCB(s *Status) mqtt.MessageHandler { - return func(client mqtt.Client, msg mqtt.Message) { - // Gather all system data metrics - memStats, err := mem.GetStats() - if err != nil { - s.Error("/stats/error", fmt.Errorf("Retrieving memory stats: %s", err)) - return - } +// StatsEvent sends statistics about resource used in the system (RAM, Disk, Network, etc...) +func (s *Status) StatsEvent(client mqtt.Client, msg mqtt.Message) { + // Gather all system data metrics + memStats, err := mem.GetStats() + if err != nil { + s.Error("/stats/error", fmt.Errorf("Retrieving memory stats: %s", err)) + return + } - diskStats, err := disk.GetStats() - if err != nil { - s.Error("/stats/error", fmt.Errorf("Retrieving disk stats: %s", err)) - return - } + diskStats, err := disk.GetStats() + if err != nil { + s.Error("/stats/error", fmt.Errorf("Retrieving disk stats: %s", err)) + return + } - type StatsPayload struct { - Memory *mem.Stats `json:"memory"` - Disk []*disk.FSStats `json:"disk"` - } + type StatsPayload struct { + Memory *mem.Stats `json:"memory"` + Disk []*disk.FSStats `json:"disk"` + } - info := StatsPayload{ - Memory: memStats, - Disk: diskStats, - } + info := StatsPayload{ + Memory: memStats, + Disk: diskStats, + } - // Send result - data, err := json.Marshal(info) - if err != nil { - s.Error("/stats/error", fmt.Errorf("Json marsahl result: %s", err)) - return - } + // Send result + data, err := json.Marshal(info) + if err != nil { + s.Error("/stats/error", fmt.Errorf("Json marsahl result: %s", err)) + return + } - //var out bytes.Buffer - //json.Indent(&out, data, "", " ") - //fmt.Println(string(out.Bytes())) + //var out bytes.Buffer + //json.Indent(&out, data, "", " ") + //fmt.Println(string(out.Bytes())) - s.Info("/stats", string(data)+"\n") - } + s.Info("/stats", string(data)+"\n") } diff --git a/main.go b/main.go index a5b870c7..1c93e1e8 100644 --- a/main.go +++ b/main.go @@ -207,11 +207,11 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { if status == nil { return } - mqttClient.Subscribe("$aws/things/"+id+"/status/post", 1, StatusCB(status)) - mqttClient.Subscribe("$aws/things/"+id+"/upload/post", 1, UploadCB(status)) - mqttClient.Subscribe("$aws/things/"+id+"/sketch/post", 1, SketchCB(status)) - mqttClient.Subscribe("$aws/things/"+id+"/update/post", 1, UpdateCB(status)) - mqttClient.Subscribe("$aws/things/"+id+"/stats/post", 1, StatsCB(status)) + mqttClient.Subscribe("$aws/things/"+id+"/status/post", 1, status.StatusEvent) + mqttClient.Subscribe("$aws/things/"+id+"/upload/post", 1, status.UploadEvent) + mqttClient.Subscribe("$aws/things/"+id+"/sketch/post", 1, status.SketchEvent) + mqttClient.Subscribe("$aws/things/"+id+"/update/post", 1, status.UpdateEvent) + mqttClient.Subscribe("$aws/things/"+id+"/stats/post", 1, status.StatsEvent) } func addFileToSketchDB(file os.FileInfo, status *Status) *SketchStatus { From 9d70b7be397bb5e52ee866a2846d26dc37624e38 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 13 Dec 2017 12:41:07 +0100 Subject: [PATCH 03/25] Added status.InfoCommandOutput to simplify sending commands output --- status.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/status.go b/status.go index 171f6c4a..52600c17 100644 --- a/status.go +++ b/status.go @@ -141,6 +141,27 @@ func (s *Status) Raw(topic, msg string) { token.Wait() } +// InfoCommandOutput sends command output on the specified topic +func (s *Status) InfoCommandOutput(topic string, out []byte, err error) { + // Prepare response payload + type response struct { + Result string + Output string + } + info := response{ + Result: err.Error(), + Output: string(out), + } + data, err := json.Marshal(info) + if err != nil { + s.Error(topic+"/error", fmt.Errorf("Json marshal result: %s", err)) + return + } + + // Send result + s.Info(topic, string(data)+"\n") +} + // Publish sens on the /status topic a json representation of the connector func (s *Status) Publish() { data, err := json.Marshal(s) From 3585708597442451474df5b793b386f9e28f8a8b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 13 Dec 2017 12:41:44 +0100 Subject: [PATCH 04/25] Added APT package manager handlers --- handler_apt_packages.go | 137 ++++++++++++++++++++++++++++++++++++++++ main.go | 6 ++ 2 files changed, 143 insertions(+) create mode 100644 handler_apt_packages.go diff --git a/handler_apt_packages.go b/handler_apt_packages.go new file mode 100644 index 00000000..60d8edf2 --- /dev/null +++ b/handler_apt_packages.go @@ -0,0 +1,137 @@ +// +// This file is part of arduino-connector +// +// Copyright (C) 2017 Arduino AG (http://www.arduino.cc/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "encoding/json" + "fmt" + + apt "github.com/arduino/go-apt-client" + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// AptListEvent sends a list of available packages and their status +func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { + // Get packages from system + all, err := apt.List() + if err != nil { + s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) + return + } + + upgradable, err := apt.ListUpgradable() + if err != nil { + s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) + return + } + + // Prepare response payload + type response struct { + Packages []*apt.Package `json:"packages"` + Upgradable []*apt.Package `json:"upgradable"` + } + info := response{ + Packages: all, + Upgradable: upgradable, + } + + // Send result + data, err := json.Marshal(info) + if err != nil { + s.Error("/apt/list/error", fmt.Errorf("Json marshal result: %s", err)) + return + } + + //var out bytes.Buffer + //json.Indent(&out, data, "", " ") + //fmt.Println(string(out.Bytes())) + + s.Info("/apt/list", string(data)+"\n") +} + +// AptInstallEvent installs new packages +func (s *Status) AptInstallEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Packages []string `json:"packages"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/install/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + toInstall := []*apt.Package{} + for _, p := range params.Packages { + toInstall = append(toInstall, &apt.Package{Name: p}) + } + out, err := apt.Install(toInstall...) + s.InfoCommandOutput("/apt/install", out, err) +} + +// AptUpdateEvent checks repositories for updates on installed packages +func (s *Status) AptUpdateEvent(client mqtt.Client, msg mqtt.Message) { + out, err := apt.CheckForUpdates() + s.InfoCommandOutput("/apt/update", out, err) +} + +// AptUpgradeEvent installs upgrade for specified packages (or for all +// upgradable packages if none are specified) +func (s *Status) AptUpgradeEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Packages []string `json:"packages"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/upgrade/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + toUpgrade := []*apt.Package{} + for _, p := range params.Packages { + toUpgrade = append(toUpgrade, &apt.Package{Name: p}) + } + + if len(toUpgrade) == 0 { + out, err := apt.UpgradeAll() + s.InfoCommandOutput("/apt/upgrade", out, err) + } else { + out, err := apt.Upgrade(toUpgrade...) + s.InfoCommandOutput("/apt/upgrade", out, err) + } +} + +// AptRemoveEvent deinstall the specified packages +func (s *Status) AptRemoveEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Packages []string `json:"packages"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/remove/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + toRemove := []*apt.Package{} + for _, p := range params.Packages { + toRemove = append(toRemove, &apt.Package{Name: p}) + } + + out, err := apt.Remove(toRemove...) + s.InfoCommandOutput("/apt/remove", out, err) +} diff --git a/main.go b/main.go index 1c93e1e8..a53b3f84 100644 --- a/main.go +++ b/main.go @@ -212,6 +212,12 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { mqttClient.Subscribe("$aws/things/"+id+"/sketch/post", 1, status.SketchEvent) mqttClient.Subscribe("$aws/things/"+id+"/update/post", 1, status.UpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/stats/post", 1, status.StatsEvent) + + mqttClient.Subscribe("$aws/things/"+id+"/apt/list/post", 1, status.AptListEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/Remove/post", 1, status.AptRemoveEvent) } func addFileToSketchDB(file os.FileInfo, status *Status) *SketchStatus { From 72c1a3611dcdeac63b9983ea24c48eaf9fe1c2e6 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 13 Dec 2017 13:07:58 +0100 Subject: [PATCH 05/25] Added APT repository handlers --- handler_apt_repositories.go | 111 ++++++++++++++++++++++++++++++++++++ main.go | 5 ++ 2 files changed, 116 insertions(+) create mode 100644 handler_apt_repositories.go diff --git a/handler_apt_repositories.go b/handler_apt_repositories.go new file mode 100644 index 00000000..6df234ea --- /dev/null +++ b/handler_apt_repositories.go @@ -0,0 +1,111 @@ +// +// This file is part of arduino-connector +// +// Copyright (C) 2017 Arduino AG (http://www.arduino.cc/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "encoding/json" + "fmt" + + apt "github.com/arduino/go-apt-client" + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// AptRepositoryListEvent sends a list of available repositories +func (s *Status) AptRepositoryListEvent(client mqtt.Client, msg mqtt.Message) { + // Get packages from system + all, err := apt.ParseAPTConfigFolder("/etc/apt") + if err != nil { + s.Error("/apt/repos/list/error", fmt.Errorf("Retrieving repositories: %s", err)) + return + } + + // Send result + data, err := json.Marshal(all) + if err != nil { + s.Error("/apt/repos/list/error", fmt.Errorf("Json marshal result: %s", err)) + return + } + + //var out bytes.Buffer + //json.Indent(&out, data, "", " ") + //fmt.Println(string(out.Bytes())) + + s.Info("/apt/repos/list", string(data)+"\n") +} + +// AptRepositoryAddEvent adds a repository to the apt configuration +func (s *Status) AptRepositoryAddEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Repository *apt.Repository `json:"repository"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/repos/add/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + err = apt.AddRepository(params.Repository, "/etc/apt") + if err != nil { + s.Error("/apt/repos/add/error", fmt.Errorf("Adding repository '%s': %s", msg.Payload(), err)) + return + } + + s.InfoCommandOutput("/apt/repos/add", []byte("OK"), nil) +} + +// AptRepositoryRemoveEvent removes a repository from the apt configuration +func (s *Status) AptRepositoryRemoveEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Repository *apt.Repository `json:"repository"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/repos/remove/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + err = apt.RemoveRepository(params.Repository, "/etc/apt") + if err != nil { + s.Error("/apt/repos/remove/error", fmt.Errorf("Removing repository '%s': %s", msg.Payload(), err)) + return + } + + s.InfoCommandOutput("/apt/repos/remove", []byte("OK"), nil) +} + +// AptRepositoryEditEvent modifies a repository definition in the apt configuration +func (s *Status) AptRepositoryEditEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + OldRepository *apt.Repository `json:"old_repository"` + NewRepository *apt.Repository `json:"new_repository"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/repos/edit/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + + err = apt.EditRepository(params.OldRepository, params.NewRepository, "/etc/apt") + if err != nil { + s.Error("/apt/repos/edit/error", fmt.Errorf("Changing repository '%s': %s", msg.Payload(), err)) + return + } + + s.InfoCommandOutput("/apt/repos/edit", []byte("OK"), nil) +} diff --git a/main.go b/main.go index a53b3f84..0b952a81 100644 --- a/main.go +++ b/main.go @@ -218,6 +218,11 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/Remove/post", 1, status.AptRemoveEvent) + + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/list/post", 1, status.AptRepositoryListEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/add/post", 1, status.AptRepositoryAddEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/remove/post", 1, status.AptRepositoryRemoveEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/edit/post", 1, status.AptRepositoryEditEvent) } func addFileToSketchDB(file os.FileInfo, status *Status) *SketchStatus { From eb29080c0a5a25eb934eed01ef8c29c9cc07614d Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Wed, 13 Dec 2017 12:00:51 +0100 Subject: [PATCH 06/25] Add network control feature --- handlers_stats.go | 33 +++++++++++++++++++++++++++++---- main.go | 1 + 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/handlers_stats.go b/handlers_stats.go index b213dfb0..10e1fe9a 100644 --- a/handlers_stats.go +++ b/handlers_stats.go @@ -24,9 +24,26 @@ import ( "github.com/arduino/go-system-stats/disk" "github.com/arduino/go-system-stats/mem" + "github.com/arduino/go-system-stats/network" mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/pkg/errors" ) +// WiFiEvent tries to connect to the specified wifi network +func (s *Status) WiFiEvent(client mqtt.Client, msg mqtt.Message) { + // try registering a new wifi network + var info struct { + SSID string `json:"ssid"` + Password string `json:"password"` + } + err := json.Unmarshal(msg.Payload(), &info) + if err != nil { + s.Error("/wifi", errors.Wrapf(err, "unmarshal %s", msg.Payload())) + return + } + net.AddWirelessConnection(info.SSID, info.Password) +} + // StatsEvent sends statistics about resource used in the system (RAM, Disk, Network, etc...) func (s *Status) StatsEvent(client mqtt.Client, msg mqtt.Message) { // Gather all system data metrics @@ -42,14 +59,22 @@ func (s *Status) StatsEvent(client mqtt.Client, msg mqtt.Message) { return } + netStats, err := net.GetNetworkStats() + if err != nil { + s.Error("/stats/error", fmt.Errorf("Retrieving network stats: %s", err)) + return + } + type StatsPayload struct { - Memory *mem.Stats `json:"memory"` - Disk []*disk.FSStats `json:"disk"` + Memory *mem.Stats `json:"memory"` + Disk []*disk.FSStats `json:"disk"` + Network *net.Stats `json:"network"` } info := StatsPayload{ - Memory: memStats, - Disk: diskStats, + Memory: memStats, + Disk: diskStats, + Network: netStats, } // Send result diff --git a/main.go b/main.go index 0b952a81..80e25741 100644 --- a/main.go +++ b/main.go @@ -212,6 +212,7 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { mqttClient.Subscribe("$aws/things/"+id+"/sketch/post", 1, status.SketchEvent) mqttClient.Subscribe("$aws/things/"+id+"/update/post", 1, status.UpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/stats/post", 1, status.StatsEvent) + mqttClient.Subscribe("$aws/things/"+id+"/wifi/post", 1, status.WiFiEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/list/post", 1, status.AptListEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) From 5a28fa83967610de6c559b83693dbdd366605436 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Fri, 22 Dec 2017 11:29:16 +0100 Subject: [PATCH 07/25] Allow the connector to be compiled with development info Signed-off-by: Matteo Suppo --- arduino-connector.sh | 2 ++ install.go | 21 +++++++++++---------- main.go | 7 ++++++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/arduino-connector.sh b/arduino-connector.sh index 670475e0..e8110857 100755 --- a/arduino-connector.sh +++ b/arduino-connector.sh @@ -22,6 +22,8 @@ export TOKEN=$token export HTTP_PROXY=$http_proxy export HTTPS_PROXY=$https_proxy export ALL_PROXY=$all_proxy +export AUTHURL=$authurl +export APIURL=$apiurl echo printenv echo --------- diff --git a/install.go b/install.go index 6f6c1cd3..a1a92432 100644 --- a/install.go +++ b/install.go @@ -48,8 +48,7 @@ import ( ) const ( - rsaBits = 2048 - devicesAPI = "https://api2.arduino.cc/devices/v1" + rsaBits = 2048 ) // Install installs the program as a service @@ -65,7 +64,7 @@ func register(config Config, token string) { // Request token var err error if token == "" { - token, err = askCredentials() + token, err = askCredentials(config.AuthURL) check(err, "AskCredentials") } @@ -88,7 +87,7 @@ func register(config Config, token string) { // Request a certificate fmt.Println("Request certificate") - pem, err := requestCert(config.ID, token, csr) + pem, err := requestCert(config.APIURL, config.ID, token, csr) check(err, "requestCert") err = ioutil.WriteFile("certificate.pem", []byte(pem), 0600) @@ -96,7 +95,7 @@ func register(config Config, token string) { // Request URL fmt.Println("Request mqtt url") - config.URL, err = requestURL(token) + config.URL, err = requestURL(config.APIURL, token) check(err, "requestURL") // Write the configuration @@ -118,7 +117,7 @@ func register(config Config, token string) { fmt.Println("Setup completed") } -func askCredentials() (token string, err error) { +func askCredentials(authURL string) (token string, err error) { var user, pass string fmt.Println("Insert your arduino username") fmt.Scanln(&user) @@ -131,6 +130,8 @@ func askCredentials() (token string, err error) { pass = string(bytePassword) authClient := auth.New() + authClient.CodeURL = authURL + "/oauth2/auth" + authClient.TokenURL = authURL + "/oauth2/token" authClient.ClientID = "connector" authClient.Scopes = "iot:devices" @@ -217,7 +218,7 @@ func generateCsr(priv interface{}) ([]byte, error) { return csr, nil } -func requestCert(id, token string, csr []byte) (string, error) { +func requestCert(apiURL, id, token string, csr []byte) (string, error) { client := http.Client{ Timeout: 5 * time.Second, } @@ -227,7 +228,7 @@ func requestCert(id, token string, csr []byte) (string, error) { payload := `{"csr":"` + pemData.String() + `"}` payload = strings.Replace(payload, "\n", "\\n", -1) - req, err := http.NewRequest("POST", devicesAPI+"/"+id, strings.NewReader(payload)) + req, err := http.NewRequest("POST", apiURL+"/"+id, strings.NewReader(payload)) if err != nil { return "", err } @@ -257,12 +258,12 @@ func requestCert(id, token string, csr []byte) (string, error) { return data.Certificate, nil } -func requestURL(token string) (string, error) { +func requestURL(apiURL, token string) (string, error) { client := http.Client{ Timeout: 5 * time.Second, } - req, err := http.NewRequest("POST", devicesAPI+"/connect", nil) + req, err := http.NewRequest("POST", apiURL+"/connect", nil) if err != nil { return "", err } diff --git a/main.go b/main.go index 80e25741..20dd2855 100644 --- a/main.go +++ b/main.go @@ -50,6 +50,8 @@ type Config struct { HTTPProxy string HTTPSProxy string ALLProxy string + AuthURL string + APIURL string } func (c Config) String() string { @@ -62,6 +64,7 @@ func (c Config) String() string { } func main() { + // Read config config := Config{} var doLogin = flag.Bool("login", false, "Do the login and prints out a temporary token") @@ -75,6 +78,8 @@ func main() { flag.StringVar(&config.HTTPProxy, "http_proxy", "", "URL of HTTP proxy to use") flag.StringVar(&config.HTTPSProxy, "https_proxy", "", "URL of HTTPS proxy to use") flag.StringVar(&config.ALLProxy, "all_proxy", "", "URL of SOCKS proxy to use") + flag.StringVar(&config.AuthURL, "authurl", "https://hydra.arduino.cc", "Url of authentication server") + flag.StringVar(&config.APIURL, "apiurl", "https://api2.arduino.cc", "Url of api server") flag.Parse() @@ -83,7 +88,7 @@ func main() { check(err, "CreateService") if *doLogin { - token, err := askCredentials() + token, err := askCredentials(config.AuthURL) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) From e6df8a1362cec4549ea7c69223bc096b90df4412 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 28 Dec 2017 12:31:04 +0100 Subject: [PATCH 08/25] Handle errors --- install.go | 8 ++++---- main.go | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/install.go b/install.go index a1a92432..02dfeeb6 100644 --- a/install.go +++ b/install.go @@ -228,7 +228,7 @@ func requestCert(apiURL, id, token string, csr []byte) (string, error) { payload := `{"csr":"` + pemData.String() + `"}` payload = strings.Replace(payload, "\n", "\\n", -1) - req, err := http.NewRequest("POST", apiURL+"/"+id, strings.NewReader(payload)) + req, err := http.NewRequest("POST", apiURL+"/devices/v1/"+id, strings.NewReader(payload)) if err != nil { return "", err } @@ -241,7 +241,7 @@ func requestCert(apiURL, id, token string, csr []byte) (string, error) { } if res.StatusCode != 200 { - return "", errors.New("POST " + "/" + id + ": expected 200 OK, got " + res.Status) + return "", errors.New("POST " + apiURL + "/devices/v1/" + id + ": expected 200 OK, got " + res.Status) } body, err := ioutil.ReadAll(res.Body) @@ -263,7 +263,7 @@ func requestURL(apiURL, token string) (string, error) { Timeout: 5 * time.Second, } - req, err := http.NewRequest("POST", apiURL+"/connect", nil) + req, err := http.NewRequest("POST", apiURL+"/devices/v1/connect", nil) if err != nil { return "", err } @@ -286,7 +286,7 @@ func requestURL(apiURL, token string) (string, error) { err = json.Unmarshal(body, &data) if err != nil { - return "", err + return "", errors.Wrap(err, "unmarshal "+string(body)) } return data.URL, nil diff --git a/main.go b/main.go index 20dd2855..432d0d22 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,8 @@ func (c Config) String() string { out += "http_proxy=" + c.HTTPProxy + "\r\n" out += "https_proxy=" + c.HTTPSProxy + "\r\n" out += "all_proxy=" + c.ALLProxy + "\r\n" + out += "authurl=" + c.AuthURL + "\r\n" + out += "apiurl=" + c.APIURL + "\r\n" return out } From 30babe693fa4f573661073cb103aeeb57d419b63 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 28 Dec 2017 13:10:27 +0100 Subject: [PATCH 09/25] Don't replicate authurl and apiurl --- arduino-connector.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/arduino-connector.sh b/arduino-connector.sh index e8110857..670475e0 100755 --- a/arduino-connector.sh +++ b/arduino-connector.sh @@ -22,8 +22,6 @@ export TOKEN=$token export HTTP_PROXY=$http_proxy export HTTPS_PROXY=$https_proxy export ALL_PROXY=$all_proxy -export AUTHURL=$authurl -export APIURL=$apiurl echo printenv echo --------- From b3a12e1dcf22f1bf1a6c37c215d428e146572712 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 28 Dec 2017 18:44:04 +0100 Subject: [PATCH 10/25] Added search parameter in apt package listing --- handler_apt_packages.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index 60d8edf2..56ae1262 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -28,8 +28,20 @@ import ( // AptListEvent sends a list of available packages and their status func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { + var params struct { + Search string `json:"search"` + } + err := json.Unmarshal(msg.Payload(), ¶ms) + if err != nil { + s.Error("/apt/list/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + return + } + // Get packages from system - all, err := apt.List() + if params.Search == "" { + params.Search = "*" + } + all, err := apt.Search(params.Search) if err != nil { s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) return From 9ad37fe637958b4f42be31ad836534b2860ad8cf Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 28 Dec 2017 18:44:43 +0100 Subject: [PATCH 11/25] Small improvement in apt-package listing type response struct { Packages []*apt.Package `json:"packages"` Updates []*apt.Package `json:"updates"` } - "upgradable" status is copied in the main list `Package` - `Upgradable` list has been renamed to `Updates` that better reflect the meaning of the list --- handler_apt_packages.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index 56ae1262..4cb62a28 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -47,20 +47,30 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { return } - upgradable, err := apt.ListUpgradable() + // On upgradable packages set the status to "upgradable" + updates, err := apt.ListUpgradable() if err != nil { s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) return } + for _, update := range updates { + for i := range all { + if update.Name == all[i].Name { + all[i].Status = "upgradable" + break + } + } + } + // Prepare response payload type response struct { - Packages []*apt.Package `json:"packages"` - Upgradable []*apt.Package `json:"upgradable"` + Packages []*apt.Package `json:"packages"` + Updates []*apt.Package `json:"updates"` } info := response{ - Packages: all, - Upgradable: upgradable, + Packages: all, + Updates: updates, } // Send result From 941132e01b0bcb2d18054d1e2b17786335b8f06c Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 28 Dec 2017 20:23:57 +0100 Subject: [PATCH 12/25] Added pagination on package listing --- handler_apt_packages.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index 4cb62a28..46e884b6 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -28,8 +28,11 @@ import ( // AptListEvent sends a list of available packages and their status func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { + const ITEMS_PER_PAGE = 30 + var params struct { Search string `json:"search"` + Page int `json:"page"` } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { @@ -47,17 +50,32 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { return } - // On upgradable packages set the status to "upgradable" - updates, err := apt.ListUpgradable() + // Paginate data + pages := (len(all)-1)/ITEMS_PER_PAGE + 1 + first := params.Page * ITEMS_PER_PAGE + last := first + ITEMS_PER_PAGE + if first >= len(all) { + all = all[0:0] + } else if last >= len(all) { + all = all[first:] + } else { + all = all[first:last] + } + + // On upgradable packages set the status to "upgradable" and add the + // available upgrade to the Updates list + allUpdates, err := apt.ListUpgradable() if err != nil { s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) return } - for _, update := range updates { + updates := []*apt.Package{} + for _, update := range allUpdates { for i := range all { if update.Name == all[i].Name { all[i].Status = "upgradable" + updates = append(updates, update) break } } @@ -67,10 +85,14 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { type response struct { Packages []*apt.Package `json:"packages"` Updates []*apt.Package `json:"updates"` + Page int `json:"page"` + Pages int `json:"pages"` } info := response{ Packages: all, Updates: updates, + Page: params.Page, + Pages: pages, } // Send result From 3c4bff6238e5ba32e114463e10b28c34e7908ff3 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Fri, 22 Dec 2017 15:25:26 +0100 Subject: [PATCH 13/25] Ensure all the endpoint have the same behaviour --- handler_apt_packages.go | 12 ++++++------ handler_apt_repositories.go | 16 ++++++++-------- handlers_stats.go | 8 ++++---- status.go | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index 46e884b6..c1628715 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -46,7 +46,7 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { } all, err := apt.Search(params.Search) if err != nil { - s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) + s.Error("/apt/list", fmt.Errorf("Retrieving packages: %s", err)) return } @@ -66,7 +66,7 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { // available upgrade to the Updates list allUpdates, err := apt.ListUpgradable() if err != nil { - s.Error("/apt/list/error", fmt.Errorf("Retrieving packages: %s", err)) + s.Error("/apt/list", fmt.Errorf("Retrieving packages: %s", err)) return } @@ -98,7 +98,7 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { // Send result data, err := json.Marshal(info) if err != nil { - s.Error("/apt/list/error", fmt.Errorf("Json marshal result: %s", err)) + s.Error("/apt/list", fmt.Errorf("Json marshal result: %s", err)) return } @@ -116,7 +116,7 @@ func (s *Status) AptInstallEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/install/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/install", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } @@ -142,7 +142,7 @@ func (s *Status) AptUpgradeEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/upgrade/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/upgrade", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } @@ -167,7 +167,7 @@ func (s *Status) AptRemoveEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/remove/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/remove", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } diff --git a/handler_apt_repositories.go b/handler_apt_repositories.go index 6df234ea..32744457 100644 --- a/handler_apt_repositories.go +++ b/handler_apt_repositories.go @@ -31,14 +31,14 @@ func (s *Status) AptRepositoryListEvent(client mqtt.Client, msg mqtt.Message) { // Get packages from system all, err := apt.ParseAPTConfigFolder("/etc/apt") if err != nil { - s.Error("/apt/repos/list/error", fmt.Errorf("Retrieving repositories: %s", err)) + s.Error("/apt/repos/list", fmt.Errorf("Retrieving repositories: %s", err)) return } // Send result data, err := json.Marshal(all) if err != nil { - s.Error("/apt/repos/list/error", fmt.Errorf("Json marshal result: %s", err)) + s.Error("/apt/repos/list", fmt.Errorf("Json marshal result: %s", err)) return } @@ -56,13 +56,13 @@ func (s *Status) AptRepositoryAddEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/repos/add/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/add", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } err = apt.AddRepository(params.Repository, "/etc/apt") if err != nil { - s.Error("/apt/repos/add/error", fmt.Errorf("Adding repository '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/add", fmt.Errorf("Adding repository '%s': %s", msg.Payload(), err)) return } @@ -76,13 +76,13 @@ func (s *Status) AptRepositoryRemoveEvent(client mqtt.Client, msg mqtt.Message) } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/repos/remove/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/remove", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } err = apt.RemoveRepository(params.Repository, "/etc/apt") if err != nil { - s.Error("/apt/repos/remove/error", fmt.Errorf("Removing repository '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/remove", fmt.Errorf("Removing repository '%s': %s", msg.Payload(), err)) return } @@ -97,13 +97,13 @@ func (s *Status) AptRepositoryEditEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/repos/edit/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/edit", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } err = apt.EditRepository(params.OldRepository, params.NewRepository, "/etc/apt") if err != nil { - s.Error("/apt/repos/edit/error", fmt.Errorf("Changing repository '%s': %s", msg.Payload(), err)) + s.Error("/apt/repos/edit", fmt.Errorf("Changing repository '%s': %s", msg.Payload(), err)) return } diff --git a/handlers_stats.go b/handlers_stats.go index 10e1fe9a..ebb291ab 100644 --- a/handlers_stats.go +++ b/handlers_stats.go @@ -49,19 +49,19 @@ func (s *Status) StatsEvent(client mqtt.Client, msg mqtt.Message) { // Gather all system data metrics memStats, err := mem.GetStats() if err != nil { - s.Error("/stats/error", fmt.Errorf("Retrieving memory stats: %s", err)) + s.Error("/stats", fmt.Errorf("Retrieving memory stats: %s", err)) return } diskStats, err := disk.GetStats() if err != nil { - s.Error("/stats/error", fmt.Errorf("Retrieving disk stats: %s", err)) + s.Error("/stats", fmt.Errorf("Retrieving disk stats: %s", err)) return } netStats, err := net.GetNetworkStats() if err != nil { - s.Error("/stats/error", fmt.Errorf("Retrieving network stats: %s", err)) + s.Error("/stats", fmt.Errorf("Retrieving network stats: %s", err)) return } @@ -80,7 +80,7 @@ func (s *Status) StatsEvent(client mqtt.Client, msg mqtt.Message) { // Send result data, err := json.Marshal(info) if err != nil { - s.Error("/stats/error", fmt.Errorf("Json marsahl result: %s", err)) + s.Error("/stats", fmt.Errorf("Json marsahl result: %s", err)) return } diff --git a/status.go b/status.go index 52600c17..e891f8f6 100644 --- a/status.go +++ b/status.go @@ -154,7 +154,7 @@ func (s *Status) InfoCommandOutput(topic string, out []byte, err error) { } data, err := json.Marshal(info) if err != nil { - s.Error(topic+"/error", fmt.Errorf("Json marshal result: %s", err)) + s.Error(topic, fmt.Errorf("Json marshal result: %s", err)) return } @@ -171,7 +171,7 @@ func (s *Status) Publish() { //fmt.Println(string(out.Bytes())) if err != nil { - s.Error("/status/error", errors.Wrap(err, "status request")) + s.Error("/status", errors.Wrap(err, "status request")) return } From 717b95ba1ffc530ff12f6bd956117f43f10c9d23 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Fri, 22 Dec 2017 16:33:40 +0100 Subject: [PATCH 14/25] Docs --- README.md | 215 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e9d4661..6234d605 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,217 @@ The Arduino Connector is tied to a specific device registered within the Arduino Make sure you have an Arduino Account and you are able to log at: https://auth.arduino.cc/login -Please write us at auth@arduino.cc if you encounter any issue loggin in and you need support. \ No newline at end of file +Please write us at auth@arduino.cc if you encounter any issue loggin in and you need support. + +### API + +To control the arduino-connector you must have: + +- the ID of the device in which the arduino-connector has been installed (eg `username:0002251d-4e19-4cc8-a4a9-1de215bfb502`) +- a working mqtt connection + +Send messages to the topic ending with /post, Receive the answer from the topic ending with /. Errors are sent to the same endpoint. + +You can distinguish between errors and non-errors because of the INFO: or ERROR: prefix of the message + +#### Status + +Retrieve the status of the connector +``` +{} +--> $aws/things/{{id}}/status/post + +INFO: { + "sketches": { + "4c1f3a9d-ed78-4ae4-94c8-bcfa2e94c692": { + "name":"sketch_oct31a", + "id":"4c1f3a9d-ed78-4ae4-94c8-bcfa2e94c692", + "pid":31343, + "status":"RUNNING", + "endpoints":null + } + } +} +<-- $aws/things/{{id}}/status +``` + +Upload a sketch on the connector +``` +{ + "token": "toUZDUNTcooVlyqAUwooBGAEtgr8iPzp017RhcST8gM.bDBgrxVzKKySBX-kBPMRqFRqlP3j_cwlgt9qPh_Ct2Y", + "url": "https://api-builder.arduino.cc/builder/v1/compile/sketch_oct31a.bin", + "name": "sketch_oct31a", + "id": "4c1f3a9d-ed78-4ae4-94c8-bcfa2e94c692" +} +--> $aws/things/{{id}}/upload/post + +INFO: Sketch started with PID 570 +<-- $aws/things/{{id}}/upload +``` + +Update the arduino-connector (doesn't return anything) +``` +{ + "url": "https://downloads.arduino.cc/tools/arduino-connector", +} +--> $aws/things/{{id}}/update/post + +<-- $aws/things/{{id}}/sketch +``` + +Update the arduino-connector +``` +{ + "url": "https://downloads.arduino.cc/tools/arduino-connector", +} +--> $aws/things/{{id}}/update/post + +<-- $aws/things/{{id}}/sketch +``` + +Retrieve the stats of the machine (memory, disk, networks) +``` +{} +--> $aws/things/{{id}}/stats/post + +INFO: { + "memory":{ + "FreeMem":1317964, + "TotalMem":15859984, + "AvailableMem":8184204, + "Buffers":757412, + "Cached":6569888, + "FreeSwapMem":0, + "TotalSwapMem":0 + }, + "disk":[ + { + "Device":"sysfs", + "Type":"sysfs", + "MountPoint":"/sys", + "FreeSpace":0, + "AvailableSpace":0, + "DiskSize":0 + }, + ], + "network":{ + "Devices":[ + { + "AccessPoints":[ + { + "Flags":1, + "Frequency":2437, + "HWAddress":"58:6D:8F:8F:FD:F3", + "MaxBitrate":54000, + "Mode":"Nm80211ModeInfra", + "RSNFlags":392, + "SSID":"ssid-2g", + "Strength":80, + "WPAFlags":0 + } + ], + "AvailableConnections":[ + { + "802-11-wireless":{ + "mac-address":"QOIwy+Ef", + "mac-address-blacklist":[], + "mode":"infrastructure", + "security":"802-11-wireless-security", + "seen-bssids":[ + "58:6D:8F:8F:FD:F3" + ], + "ssid":"QkNNSWxhYnMtMmc=" + }, + "802-11-wireless-security":{ + "auth-alg":"open", + "group":[], + "key-mgmt":"wpa-psk", + "pairwise":[], + "proto":[] + }, + "connection":{ + "id":"ssid-2g", + "permissions":[], + "secondaries":[], + "timestamp":1513953989, + "type":"802-11-wireless", + "uuid":"b5dd1024-db02-4e0f-ad3b-c41c375f750a" + }, + "ipv4":{ + "address-data":[], + "addresses":[], + "dns":[], + "dns-search":[], + "method":"auto", + "route-data":[], + "routes":[] + }, + "ipv6":{ + "address-data":[], + "addresses":[], + "dns":[], + "dns-search":[], + "method":"auto", + "route-data":[], + "routes":[] + } + } + ], + "DeviceType":"NmDeviceTypeWifi", + "IP4Config":{ + "Addresses":[ + { + "Address":"10.130.22.132", + "Prefix":24, + "Gateway":"10.130.22.1" + } + ], + "Domains":[], + "Nameservers":[ + "10.130.22.1" + ], + "Routes":[] + }, + "Interface":"wlp4s0", + "State":"NmDeviceStateActivated" + } + ], + "Status":"NmStateConnectedGlobal" + } +} +<-- $aws/things/{{id}}/stats +``` + +Configure the wifi (doesn't return anything) +``` +{ + "ssid": "ssid-2g", + "password": "passwordssid" +} +--> $aws/things/{{id}}/stats/post + +<-- $aws/things/{{id}}/stats +``` + +#### Package Management + +Retrieve a list of the installed packages + +``` +{} +--> $aws/things/{{id}}/apt/list/post + +INFO: {"memory": ... ,"disk": ... } +<-- $aws/things/{{id}}/stats +``` + + mqttClient.Subscribe("$aws/things/"+id+"/apt/list/post", 1, status.AptListEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/Remove/post", 1, status.AptRemoveEvent) + + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/list/post", 1, status.AptRepositoryListEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/add/post", 1, status.AptRepositoryAddEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/remove/post", 1, status.AptRepositoryRemoveEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/edit/post", 1, status.AptRepositoryEditEvent) \ No newline at end of file From 44c30ba2890533b80970948120783a0c1af23708 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Tue, 2 Jan 2018 11:12:55 +0100 Subject: [PATCH 15/25] apt.list: return upgradable packages when search is not set --- README.md | 77 +++++++++++++++++++++++++++++------------ handler_apt_packages.go | 13 ++++--- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6234d605..08057be4 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ Retrieve the stats of the machine (memory, disk, networks) {} --> $aws/things/{{id}}/stats/post -INFO: { - "memory":{ +INFO: { + "memory":{ "FreeMem":1317964, "TotalMem":15859984, "AvailableMem":8184204, @@ -104,7 +104,7 @@ INFO: { "TotalSwapMem":0 }, "disk":[ - { + { "Device":"sysfs", "Type":"sysfs", "MountPoint":"/sys", @@ -113,11 +113,11 @@ INFO: { "DiskSize":0 }, ], - "network":{ - "Devices":[ - { - "AccessPoints":[ - { + "network":{ + "Devices":[ + { + "AccessPoints":[ + { "Flags":1, "Frequency":2437, "HWAddress":"58:6D:8F:8F:FD:F3", @@ -129,26 +129,26 @@ INFO: { "WPAFlags":0 } ], - "AvailableConnections":[ - { - "802-11-wireless":{ + "AvailableConnections":[ + { + "802-11-wireless":{ "mac-address":"QOIwy+Ef", "mac-address-blacklist":[], "mode":"infrastructure", "security":"802-11-wireless-security", - "seen-bssids":[ + "seen-bssids":[ "58:6D:8F:8F:FD:F3" ], "ssid":"QkNNSWxhYnMtMmc=" }, - "802-11-wireless-security":{ + "802-11-wireless-security":{ "auth-alg":"open", "group":[], "key-mgmt":"wpa-psk", "pairwise":[], "proto":[] }, - "connection":{ + "connection":{ "id":"ssid-2g", "permissions":[], "secondaries":[], @@ -156,7 +156,7 @@ INFO: { "type":"802-11-wireless", "uuid":"b5dd1024-db02-4e0f-ad3b-c41c375f750a" }, - "ipv4":{ + "ipv4":{ "address-data":[], "addresses":[], "dns":[], @@ -165,7 +165,7 @@ INFO: { "route-data":[], "routes":[] }, - "ipv6":{ + "ipv6":{ "address-data":[], "addresses":[], "dns":[], @@ -177,16 +177,16 @@ INFO: { } ], "DeviceType":"NmDeviceTypeWifi", - "IP4Config":{ - "Addresses":[ - { + "IP4Config":{ + "Addresses":[ + { "Address":"10.130.22.132", "Prefix":24, "Gateway":"10.130.22.1" } ], "Domains":[], - "Nameservers":[ + "Nameservers":[ "10.130.22.1" ], "Routes":[] @@ -214,17 +214,48 @@ Configure the wifi (doesn't return anything) #### Package Management -Retrieve a list of the installed packages +Retrieve a list of the upgradable packages ``` {} --> $aws/things/{{id}}/apt/list/post -INFO: {"memory": ... ,"disk": ... } +INFO: {"packages":[ + {"Name":"firefox","Status":"upgradable","Architecture":"amd64","Version":"57.0.3+build1-0ubuntu0.17.10.1"}, + {"Name":"firefox-locale-en","Status":"upgradable","Architecture":"amd64","Version":"57.0.3+build1-0ubuntu0.17.10.1"} + ], + "page":0,"pages":1} +<-- $aws/things/{{id}}/stats +``` + +Search for installed/installable/upgradable packages + +``` +{"search": "linux"} +--> $aws/things/{{id}}/apt/list/post + +INFO: {"packages":[ + {"Name":"binutils-x86-64-linux-gnu","Status":"installed","Architecture":"amd64","Version":"2.29.1-4ubuntu1"}, + {"Name":"firmware-linux","Status":"not-installed","Architecture":"","Version":""}, + ... + ],"page":0,"pages":6} <-- $aws/things/{{id}}/stats ``` - mqttClient.Subscribe("$aws/things/"+id+"/apt/list/post", 1, status.AptListEvent) +Navigate pages + +``` +{"search": "linux", "page": 2} +--> $aws/things/{{id}}/apt/list/post + +INFO: {"packages":[ + {"Name":"linux-image-4.10.0-30-generic","Status":"config-files","Architecture":"amd64","Version":"4.10.0-30.34"}, + {"Name":"linux-image-4.13.0-21-generic","Status":"installed","Architecture":"amd64","Version":"4.13.0-21.24"}, + ... + ],"page":2,"pages":6} +<-- $aws/things/{{id}}/stats + + mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index c1628715..e5360c34 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -36,15 +36,18 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { } err := json.Unmarshal(msg.Payload(), ¶ms) if err != nil { - s.Error("/apt/list/error", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) + s.Error("/apt/list", fmt.Errorf("Unmarshal '%s': %s", msg.Payload(), err)) return } // Get packages from system + var all []*apt.Package if params.Search == "" { - params.Search = "*" + all, err = apt.ListUpgradable() + } else { + all, err = apt.Search(params.Search) } - all, err := apt.Search(params.Search) + if err != nil { s.Error("/apt/list", fmt.Errorf("Retrieving packages: %s", err)) return @@ -70,12 +73,10 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { return } - updates := []*apt.Package{} for _, update := range allUpdates { for i := range all { if update.Name == all[i].Name { all[i].Status = "upgradable" - updates = append(updates, update) break } } @@ -84,13 +85,11 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { // Prepare response payload type response struct { Packages []*apt.Package `json:"packages"` - Updates []*apt.Package `json:"updates"` Page int `json:"page"` Pages int `json:"pages"` } info := response{ Packages: all, - Updates: updates, Page: params.Page, Pages: pages, } From 7b2c76d9caeaf68fd12e8ebf0f1c5a1c0d04a4c7 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 4 Jan 2018 13:56:12 +0100 Subject: [PATCH 16/25] Fixed typo in mqtt queue identifier --- README.md | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 08057be4..1d032bc1 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ INFO: {"packages":[ mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) - mqttClient.Subscribe("$aws/things/"+id+"/apt/Remove/post", 1, status.AptRemoveEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/remove/post", 1, status.AptRemoveEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/list/post", 1, status.AptRepositoryListEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/add/post", 1, status.AptRepositoryAddEvent) diff --git a/main.go b/main.go index 432d0d22..444226fe 100644 --- a/main.go +++ b/main.go @@ -225,7 +225,7 @@ func subscribeTopics(mqttClient mqtt.Client, id string, status *Status) { mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) - mqttClient.Subscribe("$aws/things/"+id+"/apt/Remove/post", 1, status.AptRemoveEvent) + mqttClient.Subscribe("$aws/things/"+id+"/apt/remove/post", 1, status.AptRemoveEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/list/post", 1, status.AptRepositoryListEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/add/post", 1, status.AptRepositoryAddEvent) From 86c107b51854aa5a1d1c759e73cfceb645abd8cc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 4 Jan 2018 13:57:35 +0100 Subject: [PATCH 17/25] Small fix in readme --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1d032bc1..1feb0bc8 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ INFO: {"packages":[ {"Name":"firefox-locale-en","Status":"upgradable","Architecture":"amd64","Version":"57.0.3+build1-0ubuntu0.17.10.1"} ], "page":0,"pages":1} -<-- $aws/things/{{id}}/stats +<-- $aws/things/{{id}}/apt/list ``` Search for installed/installable/upgradable packages @@ -239,7 +239,7 @@ INFO: {"packages":[ {"Name":"firmware-linux","Status":"not-installed","Architecture":"","Version":""}, ... ],"page":0,"pages":6} -<-- $aws/things/{{id}}/stats +<-- $aws/things/{{id}}/apt/list ``` Navigate pages @@ -253,9 +253,11 @@ INFO: {"packages":[ {"Name":"linux-image-4.13.0-21-generic","Status":"installed","Architecture":"amd64","Version":"4.13.0-21.24"}, ... ],"page":2,"pages":6} -<-- $aws/things/{{id}}/stats - +<-- $aws/things/{{id}}/apt/list +``` +TODO: +``` mqttClient.Subscribe("$aws/things/"+id+"/apt/install/post", 1, status.AptInstallEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/update/post", 1, status.AptUpdateEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/upgrade/post", 1, status.AptUpgradeEvent) @@ -264,4 +266,5 @@ INFO: {"packages":[ mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/list/post", 1, status.AptRepositoryListEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/add/post", 1, status.AptRepositoryAddEvent) mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/remove/post", 1, status.AptRepositoryRemoveEvent) - mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/edit/post", 1, status.AptRepositoryEditEvent) \ No newline at end of file + mqttClient.Subscribe("$aws/things/"+id+"/apt/repos/edit/post", 1, status.AptRepositoryEditEvent) +``` From 47848f9543afcee855ebb0ec18a8f080c5b63796 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 4 Jan 2018 14:01:51 +0100 Subject: [PATCH 18/25] Fixed comment --- handler_apt_packages.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index e5360c34..8a826561 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -65,8 +65,7 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { all = all[first:last] } - // On upgradable packages set the status to "upgradable" and add the - // available upgrade to the Updates list + // On upgradable packages set the status to "upgradable" allUpdates, err := apt.ListUpgradable() if err != nil { s.Error("/apt/list", fmt.Errorf("Retrieving packages: %s", err)) From 62227b468a2bca2ced939da838c1e2acf20ff40e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 4 Jan 2018 14:03:39 +0100 Subject: [PATCH 19/25] Fixed linter warning on const name --- handler_apt_packages.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/handler_apt_packages.go b/handler_apt_packages.go index 8a826561..39168a24 100644 --- a/handler_apt_packages.go +++ b/handler_apt_packages.go @@ -28,7 +28,7 @@ import ( // AptListEvent sends a list of available packages and their status func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { - const ITEMS_PER_PAGE = 30 + const itemsPerPage = 30 var params struct { Search string `json:"search"` @@ -54,9 +54,9 @@ func (s *Status) AptListEvent(client mqtt.Client, msg mqtt.Message) { } // Paginate data - pages := (len(all)-1)/ITEMS_PER_PAGE + 1 - first := params.Page * ITEMS_PER_PAGE - last := first + ITEMS_PER_PAGE + pages := (len(all)-1)/itemsPerPage + 1 + first := params.Page * itemsPerPage + last := first + itemsPerPage if first >= len(all) { all = all[0:0] } else if last >= len(all) { From 86ce7041059fee13bd68e07cb9acb603717b51cd Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Mon, 8 Jan 2018 17:36:47 +0100 Subject: [PATCH 20/25] add updater --- main.go | 8 ++ updater.go | 74 +++++++++++++ updater/updater.go | 259 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 updater.go create mode 100644 updater/updater.go diff --git a/main.go b/main.go index fc36c907..8231ce86 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,7 @@ import ( const ( configFile = "./arduino-connector.cfg" + version = "1.0.0" ) // Config holds the configuration needed by the application @@ -50,6 +51,8 @@ type Config struct { HTTPProxy string HTTPSProxy string ALLProxy string + updateUrl string + appName string } func (c Config) String() string { @@ -69,6 +72,9 @@ func main() { var doRegister = flag.Bool("register", false, "Registers on the cloud") var listenFile = flag.String("listen", "", "Tail given file and report percentage") var token = flag.String("token", "", "an authentication token") + flag.StringVar(&config.updateUrl, "updateUrl", "http://localhost/", "") + flag.StringVar(&config.appName, "appName", "arduino-connector", "") + flag.String(flag.DefaultConfigFlagname, "", "path to config file") flag.StringVar(&config.ID, "id", "", "id of the thing in aws iot") flag.StringVar(&config.URL, "url", "", "url of the thing in aws iot") @@ -124,6 +130,8 @@ func (p program) run() { // Note, all_proxy will not be used by any HTTP/HTTPS connections. p.exportProxyEnvVars() + updateHandler(p.Config) + // Start nats-server on localhost:4222 opts := server.Options{} opts.Port = 4222 diff --git a/updater.go b/updater.go new file mode 100644 index 00000000..3d2c79d2 --- /dev/null +++ b/updater.go @@ -0,0 +1,74 @@ +package main + +import ( + "fmt" + "log" + "os/exec" + "strings" + + "github.com/arduino/arduino-connector/updater" + "github.com/kardianos/osext" +) + +func updateHandler(config Config) { + + path, err := osext.Executable() + + if err != nil { + //c.JSON(500, gin.H{"error": err.Error()}) + return + } + + var up = &updater.Updater{ + CurrentVersion: version, + APIURL: config.updateUrl, + BinURL: config.updateUrl, + DiffURL: "", + Dir: "update/", + CmdName: config.appName, + } + + err = up.BackgroundRun() + + if err != nil { + return + } + + //c.JSON(200, gin.H{"success": "Please wait a moment while the agent reboots itself"}) + go restart(path) +} + +func restart(path string) { + log.Println("called restart", path) + // relaunch ourself and exit + // the relaunch works because we pass a cmdline in + // that has serial-port-json-server only initialize 5 seconds later + // which gives us time to exit and unbind from serial ports and TCP/IP + // sockets like :8989 + log.Println("Starting new spjs process") + + // figure out current path of executable so we know how to restart + // this process using osext + exePath, err3 := osext.Executable() + if err3 != nil { + log.Printf("Error getting exe path using osext lib. err: %v\n", err3) + } + + if path == "" { + log.Printf("exePath using osext: %v\n", exePath) + } else { + exePath = path + } + + exePath = strings.Trim(exePath, "\n") + + cmd := exec.Command(exePath) + + fmt.Println(cmd) + + err := cmd.Start() + if err != nil { + log.Printf("Got err restarting spjs: %v\n", err) + } + log.Fatal("Exited current spjs for restart") +} diff --git a/updater/updater.go b/updater/updater.go new file mode 100644 index 00000000..2c9bfb04 --- /dev/null +++ b/updater/updater.go @@ -0,0 +1,259 @@ +package updater + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/blang/semver" + "github.com/kr/binarydist" + update "gopkg.in/inconshreveable/go-update.v0" + + "github.com/kardianos/osext" +) + +// Update protocol: +// +// GET hk.heroku.com/hk/linux-amd64.json +// +// 200 ok +// { +// "Version": "2", +// "Sha256": "..." // base64 +// } +// +// then +// +// GET hkpatch.s3.amazonaws.com/hk/1/2/linux-amd64 +// +// 200 ok +// [bsdiff data] +// +// or +// +// GET hkdist.s3.amazonaws.com/hk/2/linux-amd64.gz +// +// 200 ok +// [gzipped executable data] +// +// + +const ( + plat = runtime.GOOS + "-" + runtime.GOARCH +) + +const devValidTime = 7 * 24 * time.Hour + +var errHashMismatch = errors.New("new file hash mismatch after patch") +var up = update.New() + +// Updater is the configuration and runtime data for doing an update. +// +// Note that ApiURL, BinURL and DiffURL should have the same value if all files are available at the same location. +// +// Example: +// +// updater := &selfupdate.Updater{ +// CurrentVersion: version, +// ApiURL: "http://updates.yourdomain.com/", +// BinURL: "http://updates.yourdownmain.com/", +// DiffURL: "http://updates.yourdomain.com/", +// Dir: "update/", +// CmdName: "myapp", // app name +// } +// if updater != nil { +// go updater.BackgroundRun() +// } +type Updater struct { + CurrentVersion string // Currently running version. + APIURL string // Base URL for API requests (json files). + CmdName string // Command name is appended to the ApiURL like http://apiurl/CmdName/. This represents one binary. + BinURL string // Base URL for full binary downloads. + DiffURL string // Base URL for diff downloads. + Dir string // Directory to store selfupdate state. + Info struct { + Version string + Sha256 []byte + } +} + +// BackgroundRun starts the update check and apply cycle. +func (u *Updater) BackgroundRun() error { + os.MkdirAll(u.getExecRelativeDir(u.Dir), 0777) + if err := up.CanUpdate(); err != nil { + log.Println(err) + return err + } + //self, err := osext.Executable() + //if err != nil { + // fail update, couldn't figure out path to self + //return + //} + // TODO(bgentry): logger isn't on Windows. Replace w/ proper error reports. + if err := u.update(); err != nil { + return err + } + return nil +} + +func fetch(url string) (io.ReadCloser, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + log.Errorf("bad http status from %s: %v", url, resp.Status) + return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status) + } + return resp.Body, nil +} + +func verifySha(bin []byte, sha []byte) bool { + h := sha256.New() + h.Write(bin) + return bytes.Equal(h.Sum(nil), sha) +} + +func (u *Updater) fetchAndApplyPatch(old io.Reader) ([]byte, error) { + r, err := fetch(u.DiffURL + u.CmdName + "/" + u.CurrentVersion + "/" + u.Info.Version + "/" + plat) + if err != nil { + return nil, err + } + defer r.Close() + var buf bytes.Buffer + err = binarydist.Patch(old, &buf, r) + return buf.Bytes(), err +} + +func (u *Updater) fetchAndVerifyPatch(old io.Reader) ([]byte, error) { + bin, err := u.fetchAndApplyPatch(old) + if err != nil { + return nil, err + } + if !verifySha(bin, u.Info.Sha256) { + return nil, errHashMismatch + } + return bin, nil +} + +func (u *Updater) fetchAndVerifyFullBin() ([]byte, error) { + bin, err := u.fetchBin() + if err != nil { + return nil, err + } + verified := verifySha(bin, u.Info.Sha256) + if !verified { + return nil, errHashMismatch + } + return bin, nil +} + +func (u *Updater) fetchBin() ([]byte, error) { + r, err := fetch(u.BinURL + u.CmdName + "/" + u.Info.Version + "/" + plat + ".gz") + if err != nil { + return nil, err + } + defer r.Close() + buf := new(bytes.Buffer) + gz, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + if _, err = io.Copy(buf, gz); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (u *Updater) fetchInfo() error { + r, err := fetch(u.APIURL + u.CmdName + "/" + plat + ".json") + if err != nil { + return err + } + defer r.Close() + err = json.NewDecoder(r).Decode(&u.Info) + if err != nil { + return err + } + if len(u.Info.Sha256) != sha256.Size { + return errors.New("bad cmd hash in info") + } + return nil +} + +func (u *Updater) getExecRelativeDir(dir string) string { + filename, _ := osext.Executable() + path := filepath.Join(filepath.Dir(filename), dir) + return path +} + +func (u *Updater) update() error { + path, err := osext.Executable() + if err != nil { + return err + } + old, err := os.Open(path) + if err != nil { + return err + } + defer old.Close() + + err = u.fetchInfo() + if err != nil { + log.Println(err) + return err + } + + v1, _ := semver.Make(u.Info.Version) + v2, _ := semver.Make(u.CurrentVersion) + + if v1.Compare(v2) <= 0 { + return errors.New("already at latest version") + } + bin, err := u.fetchAndVerifyPatch(old) + if err != nil { + if err == errHashMismatch { + log.Println("update: hash mismatch from patched binary") + } else { + if u.DiffURL != "" { + log.Println("update: patching binary,", err) + } + } + + bin, err = u.fetchAndVerifyFullBin() + if err != nil { + if err == errHashMismatch { + log.Println("update: hash mismatch from full binary") + } else { + log.Println("update: fetching full binary,", err) + } + return err + } + } + + // close the old binary before installing because on windows + // it can't be renamed if a handle to the file is still open + old.Close() + + err, errRecover := up.FromStream(bytes.NewBuffer(bin)) + if errRecover != nil { + log.Errorf("update and recovery errors: %q %q", err, errRecover) + return fmt.Errorf("update and recovery errors: %q %q", err, errRecover) + } + if err != nil { + return err + } + + return nil +} From 683f7f2cbc884c144d15e50a4f29938c825e10e1 Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Mon, 8 Jan 2018 17:46:53 +0100 Subject: [PATCH 21/25] update readme --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e9d4661..da88f979 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,19 @@ The Arduino Connector is tied to a specific device registered within the Arduino Make sure you have an Arduino Account and you are able to log at: https://auth.arduino.cc/login -Please write us at auth@arduino.cc if you encounter any issue loggin in and you need support. \ No newline at end of file +Please write us at auth@arduino.cc if you encounter any issue loggin in and you need support. + + +## Compile +``` +go get github.com/arduino/arduino-connector +go build -ldflags "-X main.version=$VERSION" github.com/arduino/arduino-connector +``` + +## Autoupdate +``` +go get github.com/sanbornm/go-selfupdate +./bin/go-selfupdate arduino-connector $VERSION +# scp -r public/* user@server:/var/www/files/arduino-connector +``` + From 5a57826c7b1fa4fb0a8dea96e2625a9ec6a28e28 Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Tue, 9 Jan 2018 09:42:16 +0100 Subject: [PATCH 22/25] sync with status refactor --- updater.go => handler_update.go | 3 +-- main.go | 3 +-- updater/updater.go | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) rename updater.go => handler_update.go (97%) diff --git a/updater.go b/handler_update.go similarity index 97% rename from updater.go rename to handler_update.go index 3d2c79d2..269be64b 100644 --- a/updater.go +++ b/handler_update.go @@ -10,10 +10,9 @@ import ( "github.com/kardianos/osext" ) -func updateHandler(config Config) { +func (s *Status) Update(config Config) { path, err := osext.Executable() - if err != nil { //c.JSON(500, gin.H{"error": err.Error()}) return diff --git a/main.go b/main.go index 51ee82cc..bc24b0a8 100644 --- a/main.go +++ b/main.go @@ -137,8 +137,6 @@ func (p program) run() { // Note, all_proxy will not be used by any HTTP/HTTPS connections. p.exportProxyEnvVars() - updateHandler(p.Config) - // Start nats-server on localhost:4222 opts := server.Options{} opts.Port = 4222 @@ -156,6 +154,7 @@ func (p program) run() { // Create global status status := NewStatus(p.Config.ID, nil) + status.Update(p.Config) // Setup MQTT connection mqttClient, err := setupMQTTConnection("certificate.pem", "certificate.key", p.Config.ID, p.Config.URL, status) diff --git a/updater/updater.go b/updater/updater.go index 2c9bfb04..954e4dce 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -8,13 +8,13 @@ import ( "errors" "fmt" "io" + "log" "net/http" "os" "path/filepath" "runtime" "time" - log "github.com/Sirupsen/logrus" "github.com/blang/semver" "github.com/kr/binarydist" update "gopkg.in/inconshreveable/go-update.v0" @@ -112,7 +112,6 @@ func fetch(url string) (io.ReadCloser, error) { return nil, err } if resp.StatusCode != 200 { - log.Errorf("bad http status from %s: %v", url, resp.Status) return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status) } return resp.Body, nil @@ -248,7 +247,6 @@ func (u *Updater) update() error { err, errRecover := up.FromStream(bytes.NewBuffer(bin)) if errRecover != nil { - log.Errorf("update and recovery errors: %q %q", err, errRecover) return fmt.Errorf("update and recovery errors: %q %q", err, errRecover) } if err != nil { From 6b066a9974103ea3f7c5eae7c4700e8584d15d50 Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Tue, 9 Jan 2018 10:06:09 +0100 Subject: [PATCH 23/25] change download update feed repository --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index bc24b0a8..12a025e0 100644 --- a/main.go +++ b/main.go @@ -77,7 +77,7 @@ func main() { var doRegister = flag.Bool("register", false, "Registers on the cloud") var listenFile = flag.String("listen", "", "Tail given file and report percentage") var token = flag.String("token", "", "an authentication token") - flag.StringVar(&config.updateUrl, "updateUrl", "http://localhost/", "") + flag.StringVar(&config.updateUrl, "updateUrl", "http://downloads.arduino.cc/tools/feed/", "") flag.StringVar(&config.appName, "appName", "arduino-connector", "") flag.String(flag.DefaultConfigFlagname, "", "path to config file") From 7b3fedda0f7f0435e95a10645f60e660eb6ef3ba Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Tue, 9 Jan 2018 11:23:41 +0100 Subject: [PATCH 24/25] print version number --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index 12a025e0..77ac1188 100644 --- a/main.go +++ b/main.go @@ -69,6 +69,8 @@ func (c Config) String() string { } func main() { + fmt.Println("Version: " + version) + // Read config config := Config{} From c027a5a8e91434308a5adeeebc4f84e72c7c207d Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Tue, 9 Jan 2018 11:32:50 +0100 Subject: [PATCH 25/25] version as var to allow rewrite via ldflag --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 77ac1188..d3f054f9 100644 --- a/main.go +++ b/main.go @@ -41,7 +41,10 @@ import ( const ( configFile = "./arduino-connector.cfg" - version = "1.0.0" +) + +var ( + version = "x.x.x" ) // Config holds the configuration needed by the application