diff --git a/README.md b/README.md index 442c8b1..a20bf48 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,22 @@ -# MachineLearning_SmartMeterAnalytics -Master Thesis Energy Engineering +# MachineLearning-Smart-Meter-Analytics (2018) -This project was developed was the purpose of master dissertation for Energy engineering course at Politecnico di Milano +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Ami-Solution/MachineLearning-Smart-Meter-Analytics/master) + +[![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Ami-Solution/MachineLearning-Smart-Meter-Analytics/blob/master/notebook/1_loading_data_and_visualization.ipynb) + +[nbviewer](https://nbviewer.ipython.org/github/Ami-Solution/MachineLearning-Smart-Meter-Analytics/blob/master/notebook/1_loading_data_and_visualization.ipynb) + +This repository contains the Jupyter notebooks of the intriguing physical constants ratios that are at play in Advanced Metering Infrstructure Analytics of [*Jupyter-Notebook-MachineLearning-Smart-Meter-Analytics(2018)*](https://www.packtpub.com/big-data-and-smart-meter/-first-e/ https://zenodo.org/badge/latestdoi/), by Ami-Solution, *Packt Publishing*. + +This repository is read-only: the source files are on the [Jupyter-Notebook-MachineLearning-Smart-Meter-Analytics](https://github.com/Ami-Solution/MachineLearning-Smart-Meter-Analytics) repository. + + +## Running the code in the cloud + +With [Binder](https://mybinder.org/), you can run most of the Jupyter notebooks directly from your web browser without installing anything. Just click on the `launch binder` button above. A temporary Jupyter Notebook server with all dependencies will be automatically launched in the cloud. It is not persistent: all your changes will be lost after some time. + + +Master Thesis Energy Engineering Course + +This project was developed with the purpose of using it as a master dissertation for Energy engineering course at Politecnico di Milano under the Data Analytics and Optimization for Energy applications Lab diff --git a/notebook/1_loading_data_and_visualization.ipynb b/notebook/1_loading_data_and_visualization.ipynb index 6438d73..8bafeae 100644 --- a/notebook/1_loading_data_and_visualization.ipynb +++ b/notebook/1_loading_data_and_visualization.ipynb @@ -7,6 +7,22 @@ "Load data from Pandas HDF5 files" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n", + "!pip install statsmodels\n", + "!pip install seaborn\n", + "!pip install scikit-neuralnetwork\n" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -37,9 +53,9 @@ }, "outputs": [], "source": [ - "disaggregated_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics/data/consumption/'\n", - "data_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics/data/'\n", - "weather_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics/data/weather/'" + "disaggregated_path = '../data/consumption/'\n", + "data_path = '../data/'\n", + "weather_path = '../data/weather/'" ] }, { @@ -711,7 +727,7 @@ }, "outputs": [], "source": [ - "weather = pd.DataFrame(load_weather_data(weather_path,'/Austin_weather_2014.csv','temperature',0),columns=['temperature'])" + "weather = pd.DataFrame(load_weather_data(weather_path,'Austin_weather_2014.csv','temperature',0),columns=['temperature'])" ] }, { @@ -722,7 +738,7 @@ }, "outputs": [], "source": [ - "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'/irradiance_2014_gen.csv','gen',1))\n", + "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'irradiance_2014_gen.csv','gen',1))\n", "# The irradiance data was obtained from the PV generation units from the buildings.\n", "# The system may sometimes record negative values, which makes no sense from an irradiance point of view.\n", "# so the negative values have been set to zero.\n", @@ -889,19 +905,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/2_linear_regression_prediction.ipynb b/notebook/2_linear_regression_prediction.ipynb index bb5efa6..5df7e91 100644 --- a/notebook/2_linear_regression_prediction.ipynb +++ b/notebook/2_linear_regression_prediction.ipynb @@ -19,6 +19,19 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n" + ] + }, { "cell_type": "code", "execution_count": 33, @@ -42,7 +55,7 @@ }, "outputs": [], "source": [ - "root_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics'\n", + "root_path = '../'\n", "disaggregated_path = root_path+'/data/consumption/'\n", "data_path = root_path+'/data/'\n", "weather_path = root_path+'/data/weather/'" @@ -106,9 +119,9 @@ }, "outputs": [], "source": [ - "weather = pd.DataFrame(load_weather_data(weather_path,'/Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", + "weather = pd.DataFrame(load_weather_data(weather_path,'Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", "\n", - "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'/irradiance_2014_gen.csv','gen',1))\n", + "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'irradiance_2014_gen.csv','gen',1))\n", "# The irradiance data was obtained from the PV generation units from the buildings.\n", "# The system may sometimes record negative values, which makes no sense from an irradiance point of view.\n", "# so the negative values have been set to zero.\n", @@ -374,19 +387,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/3_random_forest_prediction.ipynb b/notebook/3_random_forest_prediction.ipynb index bea9611..bffa4c0 100644 --- a/notebook/3_random_forest_prediction.ipynb +++ b/notebook/3_random_forest_prediction.ipynb @@ -25,6 +25,19 @@ "" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -48,7 +61,7 @@ }, "outputs": [], "source": [ - "root_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics'\n", + "root_path = '../'\n", "disaggregated_path = root_path+'/data/consumption/'\n", "data_path = root_path+'/data/'\n", "weather_path = root_path+'/data/weather/'" @@ -112,9 +125,9 @@ }, "outputs": [], "source": [ - "weather = pd.DataFrame(load_weather_data(weather_path,'/Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", + "weather = pd.DataFrame(load_weather_data(weather_path,'Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", "\n", - "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'/irradiance_2014_gen.csv','gen',1))\n", + "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'irradiance_2014_gen.csv','gen',1))\n", "# The irradiance data was obtained from the PV generation units from the buildings.\n", "# The system may sometimes record negative values, which makes no sense from an irradiance point of view.\n", "# so the negative values have been set to zero.\n", @@ -274,7 +287,7 @@ ], "source": [ "from IPython.display import Image\n", - "Image(filename=root_path+'/notebook/images_on_notebooks/accuracy_metric_from_S_Karatasou_et_al.png', width=400, height=400)" + "Image(filename=root_path+'notebook/images_on_notebooks/accuracy_metric_from_S_Karatasou_et_al.png', width=400, height=400)" ] }, { @@ -297,9 +310,9 @@ "source": [ "# Accuracy metrics\n", "metrics = accuracy_metrics()\n", - "print \"coefficient of variance = {:.2f}\".format(metrics.coeff_var(predictions_frame,consum_col_raw,consum_col_raw+'_predicted')*100)\n", - "print \"Mean bias error = {:.2f}\".format(metrics.mean_bias_err(predictions_frame,consum_col_raw,consum_col_raw+'_predicted')*100)\n", - "print \"R Squared = {:.3f}\".format(metrics.r2_score(predictions_frame,consum_col_raw,consum_col_raw+'_predicted'))" + "print( \"coefficient of variance = {:.2f}\".format(metrics.coeff_var(predictions_frame,consum_col_raw,consum_col_raw+'_predicted')*100))\n", + "print( \"Mean bias error = {:.2f}\".format(metrics.mean_bias_err(predictions_frame,consum_col_raw,consum_col_raw+'_predicted')*100))\n", + "print( \"R Squared = {:.3f}\".format(metrics.r2_score(predictions_frame,consum_col_raw,consum_col_raw+'_predicted')))" ] }, { @@ -384,19 +397,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/4_SVM_prediction.ipynb b/notebook/4_SVM_prediction.ipynb index 5a5bdff..fed38a2 100644 --- a/notebook/4_SVM_prediction.ipynb +++ b/notebook/4_SVM_prediction.ipynb @@ -25,6 +25,19 @@ "" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -48,7 +61,7 @@ }, "outputs": [], "source": [ - "root_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics'\n", + "root_path = '../'\n", "disaggregated_path = root_path+'/data/consumption/'\n", "data_path = root_path+'/data/'\n", "weather_path = root_path+'/data/weather/'" @@ -112,9 +125,9 @@ }, "outputs": [], "source": [ - "weather = pd.DataFrame(load_weather_data(weather_path,'/Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", + "weather = pd.DataFrame(load_weather_data(weather_path,'Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", "\n", - "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'/irradiance_2014_gen.csv','gen',1))\n", + "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'irradiance_2014_gen.csv','gen',1))\n", "# The irradiance data was obtained from the PV generation units from the buildings.\n", "# The system may sometimes record negative values, which makes no sense from an irradiance point of view.\n", "# so the negative values have been set to zero.\n", @@ -301,9 +314,9 @@ "source": [ "# Accuracy metrics\n", "metrics = accuracy_metrics()\n", - "print \"coefficient of variance = {:.2f}\".format(metrics.coeff_var(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted')*100)\n", - "print \"Mean bias error = {:.2f}\".format(metrics.mean_bias_err(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted')*100)\n", - "print \"R Squared = {:.3f}\".format(metrics.r2_score(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted'))" + "print( \"coefficient of variance = {:.2f}\".format(metrics.coeff_var(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted')*100))\n", + "print( \"Mean bias error = {:.2f}\".format(metrics.mean_bias_err(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted')*100))\n", + "print( \"R Squared = {:.3f}\".format(metrics.r2_score(predictions_frame_denorm,consum_col_raw,consum_col_raw+'_predicted')))" ] }, { @@ -391,19 +404,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/Comparison_of_models.ipynb b/notebook/Comparison_of_models.ipynb index 6e92e76..91a5ec6 100644 --- a/notebook/Comparison_of_models.ipynb +++ b/notebook/Comparison_of_models.ipynb @@ -19,6 +19,20 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n", + "!pip install tables\n" + ] + }, { "cell_type": "code", "execution_count": 141, @@ -42,7 +56,7 @@ }, "outputs": [], "source": [ - "root_path = 'C:/Users/MANOJ/ML_SmartMeterAnalytics'\n", + "root_path = '../'\n", "disaggregated_path = root_path+'/data/consumption/'\n", "data_path = root_path+'/data/'\n", "weather_path = root_path+'/data/weather/'" @@ -106,9 +120,9 @@ }, "outputs": [], "source": [ - "weather = pd.DataFrame(load_weather_data(weather_path,'/Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", + "weather = pd.DataFrame(load_weather_data(weather_path,'Austin_weather_2014.csv','temperature',0),columns=['temperature'])\n", "\n", - "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'/irradiance_2014_gen.csv','gen',1))\n", + "irradiance = pd.DataFrame(load_irradiance_data(weather_path,'irradiance_2014_gen.csv','gen',1))\n", "# The irradiance data was obtained from the PV generation units from the buildings.\n", "# The system may sometimes record negative values, which makes no sense from an irradiance point of view.\n", "# so the negative values have been set to zero.\n", @@ -345,19 +359,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/functions.ipynb b/notebook/functions.ipynb index 38f5802..5d63ad1 100644 --- a/notebook/functions.ipynb +++ b/notebook/functions.ipynb @@ -1,5 +1,19 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install pandas\n", + "!pip install matplotlib\n", + "!pip install sklearn\n", + "!pip install seaborn\n" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -224,9 +238,9 @@ "outputs": [], "source": [ "def print_metrics(cv,mbe,r2):\n", - " print \"coefficient of variance = {:.2f}\".format(cv)\n", - " print \"Mean bias error = {:.2f}\".format(mbe)\n", - " print \"R Squared = {:.3f}\".format(r2)" + " print(\"coefficient of variance = {:.2f}\".format(cv))\n", + " print(\"Mean bias error = {:.2f}\".format(mbe))\n", + " print(\"R Squared = {:.3f}\".format(r2))" ] }, { @@ -244,19 +258,19 @@ "kernelspec": { "display_name": "Python [default]", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.6.0" } }, "nbformat": 4, diff --git a/notebook/machine_learn.ipynb b/notebook/machine_learn.ipynb index 09004fa..e627b0c 100644 --- a/notebook/machine_learn.ipynb +++ b/notebook/machine_learn.ipynb @@ -1,5 +1,16 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!pip install sklearn\n" + ] + }, { "cell_type": "code", "execution_count": 18,