From f0c2fc3204c2a1375d3dbd57ac5980003c7eca47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 6 Dec 2017 08:31:54 +0100 Subject: [PATCH 01/16] FEAT: added eurostat index browser --- larray_editor/editor.py | 100 ++++++++++++++++++++++++++++++++++++- larray_editor/treemodel.py | 71 ++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 larray_editor/treemodel.py diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 6ed17a7..9db03ca 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -1,6 +1,7 @@ import io import os import re +from datetime import datetime import sys from collections.abc import Sequence from contextlib import redirect_stdout @@ -26,6 +27,7 @@ import matplotlib import matplotlib.axes import numpy as np +import pandas as pd import larray as la @@ -34,12 +36,14 @@ get_versions, get_documentation_url, urls, RecentlyUsedList) from larray_editor.arraywidget import ArrayEditorWidget from larray_editor.commands import EditSessionArrayCommand, EditCurrentArrayCommand +from larray_editor.treemodel import SimpleTreeNode, SimpleLazyTreeModel from qtpy.QtCore import Qt, QUrl, QSettings from qtpy.QtGui import QDesktopServices, QKeySequence from qtpy.QtWidgets import (QMainWindow, QWidget, QListWidget, QListWidgetItem, QSplitter, QFileDialog, QPushButton, QDialogButtonBox, QShortcut, QVBoxLayout, QGridLayout, QLineEdit, - QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton) + QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton, + QTreeView) try: from qtpy.QtWidgets import QUndoStack @@ -83,6 +87,87 @@ DISPLAY_IN_GRID = (la.Array, np.ndarray) +def num_leading_spaces(s): + i = 0 + while s[i] == ' ': + i += 1 + return i + + +def indented_df_to_treenode(df, indent=4, indented_col=0, colnames=None, header=None): + if colnames is None: + colnames = df.columns.tolist() + if header is None: + header = [name.capitalize() for name in colnames] + + root = SimpleTreeNode(None, header) + df = df[colnames] + parent_per_level = {0: root} + # iterating on the df rows directly (via df.iterrows()) is too slow + for row in df.values: + row_data = row.tolist() + indented_col_value = row_data[indented_col] + level = num_leading_spaces(indented_col_value) // indent + # remove indentation + row_data[indented_col] = indented_col_value.strip() + + parent_node = parent_per_level[level] + node = SimpleTreeNode(parent_node, row_data) + parent_node.children.append(node) + parent_per_level[level + 1] = node + return root + + +class EurostatBrowserDialog(QDialog): + def __init__(self, index, parent=None): + super(EurostatBrowserDialog, self).__init__(parent) + + assert isinstance(index, pd.DataFrame) + + # drop unused/redundant "type" column + index = index.drop('type', axis=1) + + # display dates using locale + # import locale + # locale.setlocale(locale.LC_ALL, '') + # datetime.date.strftime('%x') + + # CHECK: this builds the whole (model) tree in memory eagerly, so it is not lazy despite using a + # Simple*Lazy*TreeModel, but it is fast enough for now. *If* it ever becomes a problem, + # we could make this lazy pretty easily (see treemodel.LazyDictTreeNode for an example). + root = indented_df_to_treenode(index) + model = SimpleLazyTreeModel(root) + tree = QTreeView() + tree.setModel(model) + tree.setUniformRowHeights(True) + tree.selectionModel().currentChanged.connect(self.view_eurostat_indicator) + tree.setColumnWidth(0, 320) + + self.resize(450, 600) + self.setWindowTitle("Select dataset") + + # set the layout + layout = QVBoxLayout() + # layout.addWidget(toolbar) + layout.addWidget(tree) + self.setLayout(layout) + + def view_eurostat_indicator(self, index): + from larray_eurostat import eurostat_get + + node = index.internalPointer() + title, code, last_update_of_data, last_table_structure_change, data_start, data_end = node.data + if not node.children: + last_update_of_data = datetime.strptime(last_update_of_data, "%d.%m.%Y") + last_table_structure_change = datetime.strptime(last_table_structure_change, "%d.%m.%Y") + last_change = max(last_update_of_data, last_table_structure_change) + try: + arr = eurostat_get(code, maxage=last_change, cache_dir='__array_cache__') + except Exception: + QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) + self.parent().view_expr(arr, expr=code) + + class AbstractEditor(QMainWindow): """Abstract Editor Window""" @@ -553,6 +638,7 @@ def _setup_file_menu(self, menu_bar): # ============= # file_menu.addSeparator() file_menu.addAction(create_action(self, _('&Load Example Dataset'), triggered=self.load_example)) + file_menu.addAction(create_action(self, _('&Browse Eurostat Datasets'), triggered=self.browse_eurostat)) # ============= # # SCRIPTS # # ============= # @@ -675,7 +761,7 @@ def line_edit_update(self): def view_expr(self, array, expr): self._listwidget.clearSelection() - self.set_current_array(array, expr) + self.set_current_array(array, name=expr) def _display_in_grid(self, k, v): return not k.startswith('__') and isinstance(v, DISPLAY_IN_GRID) @@ -1153,6 +1239,16 @@ def load_example(self): filepath = AVAILABLE_EXAMPLE_DATA[dataset_name] self._open_file(filepath) + def browse_eurostat(self): + from larray_eurostat import get_index + try: + df = get_index(cache_dir='__array_cache__', maxage=None) + except: + QMessageBox.critical(self, "Error", "Failed to fetch Eurostat dataset index") + return + dialog = EurostatBrowserDialog(df, parent=self) + dialog.show() + class ArrayEditor(AbstractEditor): """Array Editor Dialog""" diff --git a/larray_editor/treemodel.py b/larray_editor/treemodel.py new file mode 100644 index 0000000..bd2831c --- /dev/null +++ b/larray_editor/treemodel.py @@ -0,0 +1,71 @@ +from qtpy.QtCore import Qt, QModelIndex, QAbstractItemModel + + +class SimpleTreeNode(object): + __slots__ = ['parent', 'data', 'children'] + + def __init__(self, parent, data): + self.parent = parent + self.data = data + self.children = [] + + +class SimpleLazyTreeModel(QAbstractItemModel): + def __init__(self, root, parent=None): + super(SimpleLazyTreeModel, self).__init__(parent) + assert isinstance(root, SimpleTreeNode) + self.root = root + + def columnCount(self, index): + node = index.internalPointer() if index.isValid() else self.root + return len(node.data) + + def data(self, index, role): + if not index.isValid(): + return None + + if role != Qt.DisplayRole: + return None + + node = index.internalPointer() + return node.data[index.column()] + + def flags(self, index): + if not index.isValid(): + return Qt.NoItemFlags + + return Qt.ItemIsEnabled | Qt.ItemIsSelectable + + def headerData(self, section, orientation, role): + if orientation == Qt.Horizontal and role == Qt.DisplayRole: + return self.root.data[section] + return None + + def index(self, row, column, parent_index): + if not self.hasIndex(row, column, parent_index): + return QModelIndex() + + parent_node = parent_index.internalPointer() if parent_index.isValid() else self.root + child_node = parent_node.children[row] + return self.createIndex(row, column, child_node) + + def parent(self, index): + if not index.isValid(): + return QModelIndex() + + child_node = index.internalPointer() + parent_node = child_node.parent + + if parent_node == self.root: + return QModelIndex() + + grand_parent = parent_node.parent + parent_row = grand_parent.children.index(parent_node) + return self.createIndex(parent_row, 0, parent_node) + + def rowCount(self, index): + if index.column() > 0: + return 0 + + node = index.internalPointer() if index.isValid() else self.root + return len(node.children) From 9dee80b517af1639ee6b5aba67b8be78959d2daf Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 14:03:43 +0200 Subject: [PATCH 02/16] implement FrequencyFilterDialog to manage frequency filtering --- larray_editor/editor.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 9db03ca..9082290 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -118,6 +118,40 @@ def indented_df_to_treenode(df, indent=4, indented_col=0, colnames=None, header= return root +class FrequencyFilterDialog(QDialog): + def __init__(self, available_labels, parent=None): + super().__init__(parent) + + self.setWindowTitle("Select frequency") + layout = QVBoxLayout(self) + hbox = QHBoxLayout() + + self.checkboxes = {} + label_map = {'M': 'Month', 'Q': 'Quarter', 'A': 'Year', 'W': 'Week', 'D': 'Day'} + + # generate only relevant checkboxes, i.e. based on available_labels argument + for label in available_labels: + checkbox = QCheckBox(label_map[label], self) + checkbox.setChecked(True) + self.checkboxes[label] = checkbox + hbox.addWidget(checkbox) + + layout.addLayout(hbox) + + ok_button = QPushButton("Ok", self) + ok_button.clicked.connect(self.accept) + + layout.addWidget(ok_button) + + # Function to pass back selected frequencies to parent dialog + def get_selected_frequencies(self): + freqs = [] + for label, checkbox in self.checkboxes.items(): + if checkbox.isChecked(): + freqs.append(label) + return freqs + + class EurostatBrowserDialog(QDialog): def __init__(self, index, parent=None): super(EurostatBrowserDialog, self).__init__(parent) From d8e1eeef5a785139ea515e04ebb7762e72e77aec Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 14:19:58 +0200 Subject: [PATCH 03/16] add functions to extract leaf node data and convert to dataframe --- larray_editor/editor.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 9082290..5748de3 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -118,6 +118,24 @@ def indented_df_to_treenode(df, indent=4, indented_col=0, colnames=None, header= return root +# Recursively traverse tree and extract the data *only* of leaf nodes +def traverse_tree(node, rows): + # If the node has no children, append its data to rows + if not node.children: + rows.append(node.data) + # If the node has children, recursively call the function for each child + for child in node.children: + traverse_tree(child, rows) + + +# Put all the leaf nodes of tree into a dataframe structure +def tree_to_dataframe(root): + rows = [] + traverse_tree(root, rows) + return pd.DataFrame(rows, columns=root.data) + + + class FrequencyFilterDialog(QDialog): def __init__(self, available_labels, parent=None): super().__init__(parent) From 82cc512aa3b60707944dc3d6f0f2400d11c27a30 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 14:51:09 +0200 Subject: [PATCH 04/16] add new UI elements to EurostatBrowserDialog (search, advanced import) --- larray_editor/editor.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 5748de3..9d75545 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -188,20 +188,44 @@ def __init__(self, index, parent=None): # Simple*Lazy*TreeModel, but it is fast enough for now. *If* it ever becomes a problem, # we could make this lazy pretty easily (see treemodel.LazyDictTreeNode for an example). root = indented_df_to_treenode(index) + self.df = tree_to_dataframe(root) + + # Create the tree view UI in Dialog model = SimpleLazyTreeModel(root) tree = QTreeView() tree.setModel(model) tree.setUniformRowHeights(True) - tree.selectionModel().currentChanged.connect(self.view_eurostat_indicator) + tree.doubleClicked.connect(self.view_eurostat_indicator) tree.setColumnWidth(0, 320) - + tree.setContextMenuPolicy(Qt.CustomContextMenu) + tree.customContextMenuRequested.connect(self.show_context_menu) + self.tree = tree + + # Create the search results list (and hide as default) + self.search_results_list = QListWidget() + self.search_results_list.itemClicked.connect(self.handle_search_item_click) + self.search_results_list.hide() + + # Create the search input field + self.search_input = QLineEdit(self) + self.search_input.setPlaceholderText("Search...") + self.search_input.textChanged.connect(self.handle_search) + + # Create the "Advanced Import" button + self.advanced_button = QPushButton("Import from Configuration File", self) + self.advanced_button.setFocusPolicy(Qt.NoFocus) # turn off + self.advanced_button.clicked.connect(self.showAdvancedPopup) + + # General settings: resize + title self.resize(450, 600) self.setWindowTitle("Select dataset") - # set the layout + # Add widgets to layout layout = QVBoxLayout() - # layout.addWidget(toolbar) + layout.addWidget(self.search_input) layout.addWidget(tree) + layout.addWidget(self.search_results_list) + layout.addWidget(self.advanced_button) self.setLayout(layout) def view_eurostat_indicator(self, index): From ecabf6d279268f321e963e791e2d058ca2e51ac4 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 15:28:19 +0200 Subject: [PATCH 05/16] given frequencies, subset a given larray via labels on its time axis --- larray_editor/editor.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 9d75545..615a66b 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -135,6 +135,35 @@ def tree_to_dataframe(root): return pd.DataFrame(rows, columns=root.data) +# Extract frequencies from given dataset and subset accordingly +def freq_eurostat(freqs, la_data): + # If "TIME_PERIOD" exists in la_data's axes names, rename it to "time" + if "TIME_PERIOD" in la_data.axes.names: + freq_data = la_data.rename(TIME_PERIOD='time') + + a_time = [] + + for freq in freqs: + # str() because larray labels are not always strings, might also return ints + if freq == 'A': + a_time += [t for t in freq_data.time.labels if '-' not in str(t)] + elif freq == 'Q': + a_time += [t for t in freq_data.time.labels if 'Q' in str(t)] + elif freq == 'M': + a_time += [t for t in freq_data.time.labels if '-' in t and 'Q' not in str(t)] + + # Maintain order and use set for non-duplicates + a_time = sorted(set(a_time)) + + if len(freqs) == 1 and freq[0] in ['A', 'Q', 'M']: + freq_value = str(freqs[0]) + else: + freq_value = freqs + + # Return with row and colum-wise subsetting + return freq_data[freq_value, a_time] + + class FrequencyFilterDialog(QDialog): def __init__(self, available_labels, parent=None): From cd19225fa7b4dac0babfa50a721eb9934a3669dd Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 15:39:17 +0200 Subject: [PATCH 06/16] integrate freq_selection and filterdialog functionality in view_eurostat_indicators --- larray_editor/editor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 615a66b..dc52df8 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -268,6 +268,18 @@ def view_eurostat_indicator(self, index): last_change = max(last_update_of_data, last_table_structure_change) try: arr = eurostat_get(code, maxage=last_change, cache_dir='__array_cache__') + current_dataset_labels = arr.freq.labels + # Present frequency popup only if there are multiple frequencies + if len(current_dataset_labels) > 1: + dialog = FrequencyFilterDialog(current_dataset_labels, self) # first argument so that only relevant labels appear in popup + result = dialog.exec_() + if result == QDialog.Accepted: + selected_frequencies = dialog.get_selected_frequencies() + else: + selected_frequencies = current_dataset_labels + + arr = freq_eurostat(selected_frequencies, arr) + except Exception: QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) self.parent().view_expr(arr, expr=code) From 31a31accd3bbe27bfc8a00f2a1792fefdccb30cc Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 15:44:24 +0200 Subject: [PATCH 07/16] view_eurostat_indicator: add object to editor and expose to shell --- larray_editor/editor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index dc52df8..a50305e 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -280,6 +280,13 @@ def view_eurostat_indicator(self, index): arr = freq_eurostat(selected_frequencies, arr) + # Load the data into the editor + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + self.parent().kernel.shell.user_ns[code] = arr + self.accept() except Exception: QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) self.parent().view_expr(arr, expr=code) From a9f29802df1ab52cc8cddbd84bbb8e1d295f82ca Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 15:53:03 +0200 Subject: [PATCH 08/16] context menu "Add to List": selection added to editor and shell --- larray_editor/editor.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index a50305e..8774e6f 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -257,6 +257,40 @@ def __init__(self, index, parent=None): layout.addWidget(self.advanced_button) self.setLayout(layout) + def show_context_menu(self, position): + menu = QMenu(self) + + # Only show the "Add to list" UI element if the current node does not have children + index = self.tree.currentIndex() + node = index.internalPointer() + if not node.children: + add_to_list_action = QAction("Add to list", self) + add_to_list_action.triggered.connect(self.add_to_list) + menu.addAction(add_to_list_action) + + # Display the context menu at the position of the mouse click + global_position = self.tree.viewport().mapToGlobal(position) + menu.exec_(global_position) + + + def add_to_list(self): + from larray_eurostat import eurostat_get + index = self.tree.currentIndex() + node = index.internalPointer() + + if not node.children: # should only work on leaf nodes + # custom tree model where each node was represented by tuple (name, code,...) => data[1] = code + code = self.tree.currentIndex().internalPointer().data[1] + arr = eurostat_get(code, cache_dir='__array_cache__') + + # Add indicator to the list + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + editor.kernel.shell.user_ns[code] = arr + + def view_eurostat_indicator(self, index): from larray_eurostat import eurostat_get From 9c6c2bbae52592ff0a87e579b81050486559786d Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 16:04:25 +0200 Subject: [PATCH 09/16] implement logic for opening and (properly) closing AdvancedPopup --- larray_editor/editor.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 8774e6f..e715300 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -323,7 +323,18 @@ def view_eurostat_indicator(self, index): self.accept() except Exception: QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) - self.parent().view_expr(arr, expr=code) + + + def showAdvancedPopup(self): + popup = AdvancedPopup(self) + # Next Line takes into account 'finished signal' of the AdvancedPopup instance and links it to closeDialog. + # In other words: closing the advancedpopup dialog box shall also close the primary dialog box. + popup.finished.connect(self.closeDialog) + popup.exec_() + + def closeDialog(self): + self.close() + class AbstractEditor(QMainWindow): From 8bc2619b584931842f380a97f62a92fef99e85e2 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 16:19:42 +0200 Subject: [PATCH 10/16] logic to handle search request and show results --- larray_editor/editor.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index e715300..452f75b 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -325,6 +325,45 @@ def view_eurostat_indicator(self, index): QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) + def handle_search(self, text): + + def search_term_filter_df(df, text): + terms = text.split() + mask = None + + # First time loop generates a sequence of booleans (lenght = nrows of df) + # Second time it takes 'boolean intersection' with previous search result(s) + for term in terms: + term_mask = (df['Code'].str.contains(term, case=False, na=False)) | (df['Title'].str.contains(term, case=False, na=False)) + if mask is None: + mask = term_mask + else: + mask &= term_mask + return df[mask] + + if text: + # when text is entered, then hide the treeview and show (new) search results + self.tree.hide() + self.search_results_list.clear() + + # filter dataframe based on search term(s) + filtered_df = search_term_filter_df(self.df, text) + + # drop duplicates in search result (i.e. same code, originating from different locations tree) + filtered_df = filtered_df.drop_duplicates(subset='Code') + + # reduce dataframe to list of strings (only retain 'title' and 'code') + results = [f"{row['Title']} ({row['Code']})" for _, row in filtered_df.iterrows()] + + self.search_results_list.addItems(results) + self.search_results_list.show() + + else: # if search field is empty + self.tree.show() + self.search_results_list.hide() + + + def showAdvancedPopup(self): popup = AdvancedPopup(self) # Next Line takes into account 'finished signal' of the AdvancedPopup instance and links it to closeDialog. From 9f684e917476f0a702fbd3bad07d46908fc36ea7 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 16:21:20 +0200 Subject: [PATCH 11/16] logic to load dataset from search results --- larray_editor/editor.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 452f75b..92e020f 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -363,6 +363,28 @@ def search_term_filter_df(df, text): self.search_results_list.hide() + def handle_search_item_click(self, item): + # Extract all relevant text within last parentheses; BUT only need last parentheses, cf. "my country (Belgium) subject X (code)" + # Assumes the (code) is always contained in last parentheses + matches = re.findall(r'\(([^)]+)\)', item.text()) + + if matches: + last_match = matches[-1] + + # Use the code extracted text from the clicked item (this ought to be correct dataset code) + code = last_match + + # Use 'code' to load the dataset via eurostat_get + from larray_eurostat import eurostat_get + arr = eurostat_get(code, cache_dir='__array_cache__') + + # Add loaded dataset to editor and shell + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + editor.kernel.shell.user_ns[code] = arr + def showAdvancedPopup(self): popup = AdvancedPopup(self) From 706e81b0c286fe056cfab1a1ece5b34a28f1b494 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 16:23:09 +0200 Subject: [PATCH 12/16] load dataset in UI via enter/return key --- larray_editor/editor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 92e020f..dee7697 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -257,6 +257,7 @@ def __init__(self, index, parent=None): layout.addWidget(self.advanced_button) self.setLayout(layout) + def show_context_menu(self, position): menu = QMenu(self) @@ -325,6 +326,12 @@ def view_eurostat_indicator(self, index): QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) + def keyPressEvent(self, event): + if event.key() in [Qt.Key_Return, Qt.Key_Enter] and self.tree.hasFocus(): + self.view_eurostat_indicator(self.tree.currentIndex()) + super(EurostatBrowserDialog, self).keyPressEvent(event) + + def handle_search(self, text): def search_term_filter_df(df, text): From 77e8a390f2f730d84de1da7443028b51ddaf7889 Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 17:08:27 +0200 Subject: [PATCH 13/16] AdvancedPopup: initializer and UI layout --- larray_editor/editor.py | 75 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index dee7697..e5146e9 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -400,11 +400,86 @@ def showAdvancedPopup(self): popup.finished.connect(self.closeDialog) popup.exec_() + def closeDialog(self): self.close() +class AdvancedPopup(QDialog): + def __init__(self, parent=None): + # Self-commentary: an additional/extra parent argument with default value None is added. + # This subtletly relates to following line of code elsewhere: + # >> popup = AdvancedPopup(self) + # The explicit 'self' argument here corresponds to the second 'parent' of AdvancedPopup (not the implicit first 'self' to be used inside the class) + # Put differently, the 'self' written in the expression 'AdvancedPopup(self)' is *at the location of that expression* actually referring to EurostatBrowserDialog + + # Pass the parent to the base class constructor + super().__init__(parent) + + # Inititalize the UI and set default values + self.initUI() + self.yaml_content = {} + + + def initUI(self): + self.resize(600, 600) + layout = QVBoxLayout(self) + + # Button to load YAML + self.loadButton = QPushButton("Load YAML", self) + self.loadButton.clicked.connect(self.loadYAML) + layout.addWidget(self.loadButton) + + # TextEdit for YAML content + self.yamlTextEdit = QTextEdit(self) + + import textwrap + default_yaml = textwrap.dedent("""\ + # First indicator with its specific settings. + indicator_name_1: + name: var1, var2, var3 + var1: + label1: renamed_label1 + label2: renamed_label2 + var2: + label1: renamed_label1 + label2: renamed_label2 + ... + # You can continue adding more indicators and labels as per your requirements. + """) + + # Set default YAML content (example) + self.yamlTextEdit.setPlainText(default_yaml) + layout.addWidget(self.yamlTextEdit) + + # Checkbox for generating a copy + self.generateCopyCheckBox = QCheckBox("Generate additional copy", self) + layout.addWidget(self.generateCopyCheckBox) + + # Horizontal layout for dropdown and path selection + copyLayout = QHBoxLayout() + + # ComboBox (Drop-down) for file formats + self.fileFormatComboBox = QComboBox(self) + self.fileFormatComboBox.addItems(["xlsx", "IODE", "csv"]) + copyLayout.addWidget(self.fileFormatComboBox) + + # Button for path selection + self.pathSelectionButton = QPushButton("Select Path...", self) + self.pathSelectionButton.clicked.connect(self.selectPath) + copyLayout.addWidget(self.pathSelectionButton) + layout.addLayout(copyLayout) + + # Final button to proceed with the main task and optionally generate a copy + self.proceedButton = QPushButton("Proceed", self) + self.proceedButton.clicked.connect(self.proceedWithTasks) + layout.addWidget(self.proceedButton) + + + + + class AbstractEditor(QMainWindow): """Abstract Editor Window""" From 3bcdf4440c8ee4ef3570e0c9d9b49a53a722398c Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 17:25:52 +0200 Subject: [PATCH 14/16] load content YAML file and adjust fileformat/path UI elements --- larray_editor/editor.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index e5146e9..0deab05 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -473,10 +473,41 @@ def initUI(self): # Final button to proceed with the main task and optionally generate a copy self.proceedButton = QPushButton("Proceed", self) - self.proceedButton.clicked.connect(self.proceedWithTasks) + self.proceedButton.clicked.connect(self.proceed_and_viewdata) layout.addWidget(self.proceedButton) + def loadYAML(self): + options = QFileDialog.Options() + filePath, _ = QFileDialog.getOpenFileName(self, "Open YAML File", "", "YAML Files (*.yaml *.yml);;All Files (*)", options=options) + if filePath: + with open(filePath, 'r') as file: + self.yaml_content = yaml.safe_load(file) + + # don't show dbdir and format in the textbox, they are already shown in the UI + self.yaml_content_subset = {k: v for k, v in self.yaml_content.items() if k not in ['dbdir', 'format']} + self.yamlTextEdit.setText(yaml.dump(self.yaml_content_subset, default_flow_style=False)) + + # Check for 'dbdir' and 'format' in the loaded content + if 'dbdir' in self.yaml_content and 'format' in self.yaml_content: + format_val = self.yaml_content['format'] + self.setComboBoxValue(format_val) + + + def setComboBoxValue(self, value): + index = self.fileFormatComboBox.findText(value) + if index != -1: # if the format from YAML exists in the ComboBox items + self.fileFormatComboBox.setCurrentIndex(index) + + + def selectPath(self): + options = QFileDialog.Options() + import os + if self.yaml_content is not None and 'dbdir' in self.yaml_content and self.yaml_content['dbdir'] is not None and os.path.exists(self.yaml_content['dbdir']) and os.path.isdir(self.yaml_content['dbdir']): + # if appropriate, then use the dbdir from the loaded YAML file as the default path in QFileDialog + self.selectedPath, _ = QFileDialog.getSaveFileName(self, "Select Path", self.yaml_content['dbdir'], "All Files (*)", options=options) + else: + self.selectedPath, _ = QFileDialog.getSaveFileName(self, "Select Path", "", "All Files (*)", options=options) From 8790d3cf26fe36950dbfc8a0f70078a89ecbc79b Mon Sep 17 00:00:00 2001 From: yvda Date: Tue, 10 Oct 2023 18:08:05 +0200 Subject: [PATCH 15/16] logic to load/export datasets on basis of YAML instructions --- larray_editor/editor.py | 111 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 0deab05..d26f7c0 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -1,6 +1,7 @@ import io import os import re +import yaml from datetime import datetime import sys from collections.abc import Sequence @@ -41,9 +42,9 @@ from qtpy.QtCore import Qt, QUrl, QSettings from qtpy.QtGui import QDesktopServices, QKeySequence from qtpy.QtWidgets import (QMainWindow, QWidget, QListWidget, QListWidgetItem, QSplitter, QFileDialog, QPushButton, - QDialogButtonBox, QShortcut, QVBoxLayout, QGridLayout, QLineEdit, + QDialogButtonBox, QShortcut, QVBoxLayout, QHBoxLayout, QGridLayout, QLineEdit, QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton, - QTreeView) + QTreeView, QTextEdit, QMenu, QAction) try: from qtpy.QtWidgets import QUndoStack @@ -473,7 +474,7 @@ def initUI(self): # Final button to proceed with the main task and optionally generate a copy self.proceedButton = QPushButton("Proceed", self) - self.proceedButton.clicked.connect(self.proceed_and_viewdata) + self.proceedButton.clicked.connect(self.proceedWithTasks) layout.addWidget(self.proceedButton) @@ -510,6 +511,110 @@ def selectPath(self): self.selectedPath, _ = QFileDialog.getSaveFileName(self, "Select Path", "", "All Files (*)", options=options) + def proceedWithTasks(self): + # Preliminary functions to extract the relevant data + def intersect(a, b): + if isinstance(a, str): + a = a.split(',') + elif isinstance(a, dict): + a = a.keys() + return [val for val in a if val in b] + + def extract_mat(mat, keys): + intersect_keys = {} + replace_keys = {} + name_keys = [] + new_keys = [] + none_keys = [] + + for k in keys: + # name: defines the order and the naming convention + if k == 'name': + name_keys = keys[k].split(',') + new_keys = [n for n in name_keys if n not in mat.axes.names] + + # select common labels + else: + intersect_keys[k] = intersect(keys[k], mat.axes[k].labels) + # prepare dict for replacement + if isinstance(keys[k], dict): + replace_keys[k] = {key: keys[k][key] for key in intersect_keys[k]} + for key, value in replace_keys[k].items(): + if value is None: + replace_keys[k][key] = key + none_keys.append(key) + + mat = mat[intersect_keys] + # replace labels + if len(replace_keys) > 0: + mat = mat.set_labels(replace_keys) + + if len(none_keys) > 0: + for nk in none_keys: + mat = mat[nk] + + # add missing dimensions + if len(new_keys) > 0: + for nk in new_keys: + mat = la.stack({nk: mat}, nk) + # put in correct order + if len(name_keys) > 0: + mat = mat.transpose(name_keys) + return mat + + if self.generateCopyCheckBox.isChecked(): + # Use UI choice as most recent info (instead of original YAML file stored in pm['dbdir'] etc.) + # Reasoning/philosophy: load YAML file, maybe later use dropdown to export other formats. + file_format = self.fileFormatComboBox.currentText() + db_dir = self.yaml_content['dbdir'] + + # There might be multiple datasets in YAML config. Redundancy to load them *all* in viewer sequentially + # Cannot see them all. So, convert dictionary to list: easier to navigate to last one (only load this). + # Note: two different tasks: 1) add datasets to Editor as available larray objects, and b) view those + # graphically in UI. Only makes sense to 'view' the last dataset if multiple indicators are in YAML, but + # all of the datasets in the YAML needed to be added to session/editor regardless. + + # Prepare list for easy access + pm = self.yaml_content + indicators_as_list = list(pm['indicators']) + + # Prepare access to already existing datasets in editor + editor = self.parent().parent() # need to be *two* class-levels higher + new_data = editor.data.copy() # make a copy of dataset-dictionary before starting to add new datasets + + from larray_eurostat import eurostat_get + for code in indicators_as_list: + # Adding datasets + arr = eurostat_get(code, cache_dir='__array_cache__') # pulls dataset + arr = extract_mat(arr, pm['indicators'][code]) # relabels dataset via YAML configs + + new_data[code] = arr # add dataset for LArrayEditor update + editor.kernel.shell.user_ns[code] = arr # add dataset to console namespace + + # Viewing datasets: only if last indicator in YAML config + if code == indicators_as_list[-1]: + editor.view_expr(arr, expr=code) + + # Export for different file format (...) + if self.generateCopyCheckBox.isChecked(): + if file_format == 'csv': + arr.to_csv(f'{db_dir}/{code}.csv') + elif file_format == 'ui': + # la.view(s) + pass + elif file_format in ['var', 'iode']: + # to_var(arr, db_dir) + pass + elif file_format[:3] == 'xls': + arr.to_excel(f'{db_dir}/{code}.xlsx') + else: + arr.to_hdf(f'{db_dir}/{code}') + + # Update mapping outside for loop -- i.e. no need to do this update multiple times + editor.update_mapping(new_data) + self.accept() + + class AbstractEditor(QMainWindow): """Abstract Editor Window""" From bd7436b716592f3c3db2114f064b835514ba5463 Mon Sep 17 00:00:00 2001 From: yvda Date: Wed, 25 Oct 2023 17:17:50 +0200 Subject: [PATCH 16/16] AMECO integration in Editor --- larray_ameco/get_ameco.py | 116 +++++++ larray_editor/ameco_toc_tree.txt | 539 ++++++++++++++++++++++++++++++ larray_editor/editor.py | 542 ++++++++++++++++++++++++++----- larray_editor/treemodel.py | 39 +++ 4 files changed, 1161 insertions(+), 75 deletions(-) create mode 100644 larray_ameco/get_ameco.py create mode 100644 larray_editor/ameco_toc_tree.txt diff --git a/larray_ameco/get_ameco.py b/larray_ameco/get_ameco.py new file mode 100644 index 0000000..4a5355a --- /dev/null +++ b/larray_ameco/get_ameco.py @@ -0,0 +1,116 @@ +import pandas as pd +import larray as la +import zipfile +import os + +# aameco_get +ameco_input_path = 'http://ec.europa.eu/economy_finance/db_indicators/ameco/documents/' +ameco_output_path = '' +ameco_output_filename = 'ameco.csv' + + +def change_label(s): + ls = s.split('.') + nls = [] + nls.append(ls[0]) + nls.append('-'.join(ls[1:5])) + nls.append(ls[5]) + + return '.'.join(nls) + + +def read_ameco(path, ameco_file): + df = pd.read_csv(path + ameco_file, sep=';', na_values=la.nan, skipinitialspace=True) + df.drop(df.columns[len(df.columns) - 1], axis=1, inplace=True) + # change last column to numeric + df[df.columns[-1]] = df[df.columns[-1]].apply(pd.to_numeric, errors='coerce') + + df = df.drop(['COUNTRY', 'SUB-CHAPTER', 'TITLE', 'UNIT'], axis=1) + idx_columns = [x for x in df.columns if not x.isdigit()] + df_var = df.set_index(idx_columns) + la_data = la.from_frame(df_var).rename({1: 'time'}) + la_data = la_data.set_labels('CODE', change_label) + + return la_data + + +def ameco_get(indicator, path='', drop_csv=True): + ''' + + Parameters + ---------- + indicator + path + drop_csv + + Returns + ------- + + ''' + + import urllib.request + + ameco_zip = indicator + '.zip' + url = ameco_input_path + ameco_zip + + file_name, headers = urllib.request.urlretrieve(url, path + ameco_zip) + print(file_name, 'Date', headers['Date'], 'Content-Length:', headers['Content-Length']) + ziplist = unzip_ameco(path, ameco_zip) + os.remove(path + ameco_zip) + for idx, ameco_file in enumerate(ziplist): + if idx == 0: + la_data = read_ameco(path, ameco_file) + else: + la_data2 = read_ameco(path, ameco_file) + la_data = la_data.append('CODE', la_data2) + + if drop_csv: + os.remove(path + ameco_file) + + la_data = la_data.rename({'CODE': 'RCT'}) + return la_data + + +def unzip_ameco(path, ameco_zip): + print('Unzipping', path + ameco_zip) + namelist = [] + with zipfile.ZipFile(path + ameco_zip, 'r') as zip_ref: + zip_ref.extractall(path) + namelist = zip_ref.namelist() + + return namelist + + +def reshape_and_dump_ameco_dataset(): + import larray as la + ameco_data = ameco_get('ameco0', path='__array_cache__/') + ameco_data = ameco_data.split_axes(sep='.') + + # >>> The RCT Axis contains data of the form 'DEU.1-0-0-0.NPTD'. + # >>> Objective is to split this into three distinct axes: Var, Country, and Numo. + + # Firs handle special case where '_' interferes with our splitting logic. + # Replace 'D_W' with a placeholder 'D°°°W' + ameco_data = ameco_data.set_labels('RCT', lambda label: label.replace('D_W', 'D°°°W')) + + # ROUND 1 + # Split and reformat the string to prepare for the first split. + # The format becomes "B_C|A" in preparation for a split on '_'. + ameco_data = ameco_data.set_labels('RCT', lambda label: (lambda parts: f"{parts[1]}_{parts[2]}|{parts[0]}")(label.split('.'))) + axis_combined_NVC = ameco_data.axes[0].rename('numo_VarCountry') + axis_time = ameco_data.axes[1] + ameco_partially_cleaned = la.Array(ameco_data, axes=[axis_combined_NVC, axis_time]).split_axes(axis_combined_NVC) + + # ROUND 2 + # Modify labels from "C|A" to "C_A" to prepare for the second split on '_'. + ameco_partially_cleaned = ameco_partially_cleaned.set_labels('VarCountry', lambda x: x.replace('|', '_')) + axis_numo = ameco_partially_cleaned.axes[0] + axis_combined_VC = ameco_partially_cleaned.axes[1].rename('Var_Country') + axis_time = ameco_partially_cleaned.axes[2] + ameco_final_cleaned = la.Array(ameco_partially_cleaned, axes=[axis_numo, axis_combined_VC, axis_time]).split_axes(axis_combined_VC) + + # Restore the special case 'D°°°W' back to 'D_W'. + ameco_final_cleaned = ameco_final_cleaned.set_labels('Country', {'D°°°W': 'D_W'}) + + ameco_final_cleaned.to_hdf('__array_cache__/data.h5', 'ameco0') + print('file dumped to __array_cache__/data.h5') diff --git a/larray_editor/ameco_toc_tree.txt b/larray_editor/ameco_toc_tree.txt new file mode 100644 index 0000000..e984025 --- /dev/null +++ b/larray_editor/ameco_toc_tree.txt @@ -0,0 +1,539 @@ +> Population and employment +>> Population +>>> Total population (NPTN) +>>> Total population (National accounts) (NPTD) +>>> Population: 0 to 14 years (NPCN) +>>> Population: 15 to 64 years (NPAN) +>>> Population: 65 years and over (NPON) +>> Labour force +>>> Total labour force (Labour force statistics) (NLTN) +>>> Civilian labour force (Labour force statistics) (NLCN) +>>> Civilian employment, persons (domestic) (NECD) +>>> Civilian employment, persons (national) (NECN) +>> Employment total economy, persons +>>> Employment, persons-; total economy (National accounts) (NETN) +>>> Employment, persons-;all domestic industries (National accounts) (NETD) +>>> Number of self-employed-; total economy (National accounts) (NSTD) +>>> Employees, persons-; total economy (National accounts) (NWTN) +>>> Employees, persons-; all domestic industries (National accounts) (NWTD) +>> Employment total economy, full-time equivalents +>>> Employees, full-time equivalents-; total economy (National accounts) (FWTD) +>>> Employment, full-time equivalents-; total economy (National accounts) (FETD) +>> Unemployment +>>> Total unemployment; Member States: definition -Eurostat (NUTN) +>>> Unemployment rate, total (percentage of civilian labour force); Member States: definition Eurostat (ZUTN) +> Consumption +>> Actual individual final consumption of households +>>> Actual individual final consumption of households at current prices (UCTH) +>>> Actual individual final consumption of households at constant prices (OCTH) +>>> Price deflator actual individual final consumption of households (PCTH) +>> Private final consumption expenditure +>>> Private final consumption expenditure at current prices (UCPH) +>>> Private final consumption expenditure at constant prices (OCPH) +>>> Price deflator private final consumption expenditure (PCPH) +>> Total final consumption expenditure of general government +>>> Final consumption expenditure of general government at current prices (UCTG) +>>> Final consumption expenditure of general government at constant prices (OCTG) +>>> Price deflator total final consumption expenditure of general government (PCTG) +>> Collective consumption expenditure of general government +>>> Collective consumption of general government at current prices (UCCG) +>>> Collective consumption of general government at constant prices (OCCG) +>>> Price deflator collective consumption of general government (PCCG) +>> Individual consumption expenditure of general government +>>> Individual consumption of general government at current prices (UCIG) +>>> Individual consumption of general government at constant prices (OCIG) +>>> Price deflator individual consumption of general government (PCIG) +>> Total consumption +>>> Total consumption expenditure at current prices (UCNT) +>>> Total consumption expenditure at constant prices (OCNT) +>>> Price deflator total consumption (PCNT) +>> Consumer price index +>>> Harmonised consumer price index (1996 = 100) (ZCPIH) +>>> National consumer price index (ZCPIN) +> Capital formation and saving, total economy and sectors +>> Gross fixed capital formation +>>> Gross fixed capital formation at constant prices; total economy (OIGT) +>>> Price deflator gross fixed capital formation; total economy (PIGT) +>>> Gross fixed capital formation at current prices; general government (UIGG) +>>> Gross fixed capital formation at current prices; private sector (UIGP) +>>> Gross fixed capital formation at current prices; total economy (UIGT) +>> Net fixed capital formation +>>> Net fixed capital formation at constant prices; total economy (OINT) +>>> Net fixed capital formation at current prices; general government (UING) +>>> Net fixed capital formation at current prices; private sector (UINP) +>>> Net fixed capital formation at current prices; total economy (UINT) +>> Consumption of fixed capital +>>> Consumption of fixed capital- at constant prices; total economy; Price deflator gross fixed capital formation (OKCT) +>>> Consumption of fixed capital at current prices; general government (UKCG) +>>> Consumption of fixed capital at current prices; total economy (UKCT) +>> Change in inventories +>>> Changes in inventories and acquisitions less disposals of valuables at constant prices; total economy (OIST) +>>> Changes in inventories and acquisitions less disposals of valuables at current prices; total economy (UIST) +>> Gross capital formation +>>> Gross capital formation at constant prices; total economy (OITT) +>>> Gross capital formation at current prices; total economy (UITT) +>> Gross saving +>>> Gross national saving (USGN) +>>> Gross saving: private sector; EU-Member States: ESA 1995 (USGP) +>>> Gross saving-; private sector; EU-Member States: former definition (USGPF) +>> Net saving +>>> Net saving; general government ; EU-Member States: ESA 1995 (USNG) +>>> Net saving: general government; EU-Member States: former definition (USNGF) +>>> Net national saving (USNN) +>>> Net saving-; private sector; EU-Member States: ESA 1995 (USNP) +>>> Net saving; private sector; EU-Member States: former definition (USNPF) +> Gross fixed capital formation by type of goods +>> Construction +>>> Gross fixed capital formation at constant prices; construction (OIGCO) +>>> Price deflator gross fixed capital formation; construction (PIGCO) +>>> Gross fixed capital formation at current prices; construction (UIGCO) +>> Dwellings +>>> Gross fixed capital formation at constant prices; dwellings (OIGDW) +>>> Price deflator gross fixed capital formation; dwellings (PIGDW) +>>> Gross fixed capital formation at current prices; dwellings (UIGDW) +>> Non-residential construction and civil engineering +>>> Gross fixed capital formation at constant prices-; non-residential construction and civil engineering (OIGNR) +>>> Price deflator gross fixed capital formation-; non-residential construction and civil engineering (PIGNR) +>>> Gross fixed capital formation at current prices-; non-residential construction and civil engineering (UIGNR) +>> Equipment +>>> Gross fixed capital formation at constant prices-; equipment (OIGEQ) +>>> Price deflator gross fixed capital formation-; equipment (PIGEQ) +>>> Gross fixed capital formation at current prices; equipment (UIGEQ) +>> Metal products and machinery +>>> Gross fixed capital formation at constant prices; metal products and machinery (OIGMA) +>>> Price deflator gross fixed capital formation; metal products and machinery (PIGMA) +>>> Gross fixed capital formation at current prices; metal products and machinery (UIGMA) +>> Transport equipment +>>> Gross fixed capital formation at constant prices transport equipment (OIGTR) +>>> Price deflator gross fixed capital formation; transport equipment (PIGTR) +>>> Gross fixed capital formation at current prices; transport equipment (UIGTR) +> Domestic and final demand +>> Domestic demand excluding stocks +>>> Domestic demand excluding stocks at constant prices (OUNF) +>>> Price deflator domestic demand excluding stocks (PUNF) +>>> Domestic demand excluding stocks at current prices (UUNF) +>> Domestic demand including stocks +>>> Domestic demand including stocks at constant prices (OUNT) +>>> Price deflator domestic demand including stocks (PUNT) +>>> Domestic demand including stocks at current prices (UUNT) +>>> Domestic demand including stocks at constant prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OUNTQ) +>>> Domestic demand including stocks at constant prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OUNTQ) +>> Final demand +>>> Final demand at constant prices (OUTT) +>>> Price deflator final demand (PUTT) +>>> Final demand at current prices (UUTT) +>> Contributions to the change of the deflator of final demand +>>> Contribution to the change of the final demand deflator of import prices (YPUT0) +>>> Contribution to the change of the final demand deflator of import prices excluding nominal effective exchange rates (YPUT1) +>>> Contribution to the change of the final demand deflator of nominal effective exchange rates (YPUT2) +>>> Contribution to the change of the final demand deflator of relative unit labour costs in national currency (YPUT3) +>>> Contribution to the change of the final demand deflator of domestic factors (YPUT5) +>>> Contribution to the change of the final demand deflator of the GDP price deflator (YPUT6) +>>> Contribution to the change of the final demand deflator of nominal unit labour costs (YPUT7) +>>> Contribution to the change of the final demand deflator of net indirect taxes (YPUT8) +>>> Contribution to the change of the final demand deflator of gross operating surplus; excluding imputed compensation of self-employed (YPUT9) +> National product and income +>> Gross national income +>>> Gross national -income at constant market prices, deflator GDP (OVGN) +>>> Gross national -income at current market prices (UVGN) +>> Gross national disposable income +>>> Gross national disposable income (UVGT) +>> Net national income +>>> National income at current market prices (UVNN) +>> Net national disposable income +>>> National disposable income [net] (UVNT) +>> Domestic product and income, total economy +>>> Gross domestic product (GDP) +>>> Gross domestic product at constant market prices (OVGD) +>>> Price deflator gross domestic product at market prices (PVGD) +>>> Gross domestic product at current market prices (UVGD) +>>> Gross domestic product at current factor cost (UYGD) +>>> Gross domestic product at constant market prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OVGDQ) +>>> Price deflator gross domestic product at market prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PVGDQ) +>>> Gross domestic product at constant market prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OVGDQ) +>>> Price deflator gross domestic product at market prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PVGDQ) +>>> GDP (reference level for excessive deficit procedure) +>>> Gross domestic product at current market prices ; Reference level for excessive deficit procedure (UVGDH) +>> GDP adjusted for the impact of terms of trade +>>> Gross domestic product at constant market prices adjusted for the impact of terms of trade (OVGDA) +>> Trend GDP at constant market prices +>>> Trend gross domestic product at constant market prices (OVGDT) +>>> Gap between actual and trend gross domestic product at constant market prices (AVGDGT) +>> Potential GDP at constant market prices +>>> Potential gross domestic product at constant market prices (OVGDP) +>>> Gap between actual and potential gross domestic product at constant market prices (AVGDGP) +>> Contributions to the change of GDP +>>> Contribution to the increase of GDP at constant market prices of private consumption (CVGD0) +>>> Contribution to the increase of GDP at constant market prices of public consumption (CVGD1) +>>> Contribution to the increase of GDP at constant market prices of total consumption (CVGD10) +>>> Contribution to the increase of GDP at constant market prices of gross fixed capital formation (CVGD2) +>>> Contribution to the increase of GDP at constant market prices of domestic demand excluding stocks (CVGD3) +>>> Contribution to the increase of GDP at constant market prices of net stockbuilding (CVGD4) +>>> Contribution to the increase of GDP at constant market prices of domestic demand including stocks (CVGD5) +>>> Contribution to the increase of GDP at constant market prices of exports of goods and services ; excluding intra-EU trade (CVGD6A) +>>> Contribution to the increase of GDP at constant market prices of final demand ; excluding intra-EU trade (CVGD7A) +>>> Contribution to the increase of GDP at constant market prices of imports of goods and services ; excluding intra-EU trade (CVGD8A) +>>> Contribution to the increase of GDP at constant market prices of the balance of goods and services (CVGD9) +>> Net domestic product +>>> Domestic income at current market prices (UVND) +>>> Domestic income at current factor cost (UYND) +>> Gross value added excluding FISIM +>>> Gross value added at constant basic prices excluding FISIM-; total economy (OVGE) +>>> Gross value added at current basic prices excluding FISIM-; total economy (UVGE) +>> Taxes linked to imports and production and subsidies +>>> Taxes linked to imports and production minus subsidies-; total economy (UTVN) +>>> Taxes linked to imports and production-; total economy (UTVT) +>>> Subsidies-; total economy (UYVT) +>> Gross operating surplus +>>> Gross operating surplus-; total economy (UOGD) +>>> Gross operating surplus-; total economy; adjusted for imputed compensation of self-employed (UQGD) +>> Net operating surplus +>>> Net operating surplus-; total economy (UOND) +>>> Net operating surplus-; total economy-; adjusted for imputed compensation of self-employed (UQND) +> Gross domestic product per head +>> GDP at current market prices per head of population +>>> Gross domestic product at current market prices per head of population (HVGDP) +>>> Gross domestic product at current market prices per head of population (HVGDPR) +>> GDP at current market prices per person employed +>>> Gross domestic product at current market prices per person employed (HVGDE) +>> GDP at constant market prices per head of population +>>> Gross domestic product at constant market prices per head of population (RVGDP) +>> GDP at constant market prices per person employed +>>> Gross domestic product at constant market prices per person employed (RVGDE) +>>> Gross domestic product at constant market prices per person employed ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (RVGDEQ) +>>> Gross domestic product at constant market prices per person employed ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (RVGDEQ) +>> GDP at constant market prices (adjusted for the impact of terms of trade) per head of population +>>> GDP at constant market prices adjusted for the impact of terms of trade per head of population (RVGDAP) +>> GDP at constant market prices (adjusted for the impact of terms of trade) per person employed +>>> GDP at constant market prices adjusted for the impact of terms of trade per person employed (RVGDAE) +> Labour costs; total economy +>> Compensation of employees +>>> Compensation of employees; total economy (UWCD) +>> Nominal compensation per employee +>>> Nominal compensation per employee: total economy (HWCDW) +>>> Nominal compensation per employee: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (HWCDWQ) +>>> Nominal compensation per employee: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (HWCDWQ) +>> Real compensation per employee +>>> Real compensation per employee, deflator private consumption; total economy (RWCDC) +>>> Real compensation per employee, deflator GDP; total economy (RWCDV) +>>> Real compensation per employee, deflator GDP: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (RWCDVQ) +>>> Real compensation per employee, deflator GDP: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (RWCDVQ) +>> Adjusted wage share +>>> Adjusted wage share total economy (ALCD0) +>>> Adjusted wage share; total economy (ALCD2) +>> Nominal unit labour costs +>>> Nominal unit labour costs; total economy (PLCD) +>>> Nominal unit labour costs: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PLCDQ) +>>> Nominal unit labour costs: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PLCDQ) +>> Real unit labour costs +>>> Real unit labour costs total economy (QLCD) +>>> Real unit labour costs: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (QLCDQ) +>>> Real unit labour costs: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (QLCDQ) +> Capital stock +>> Net capital stock at constant prices; total economy +>>> Net capital stock at constant prices; total economy (OKND) +>> Net capital stock at constant prices per person employed +>>> Net capital stock at constant prices per person employed; total economy (Capital intensity) (RKNDE) +>> GDP at constant market prices per unit of net capital stock; total economy +>>> Gross domestic product at constant market prices per unit of net capital stock (Capital productivity) (AVGDK ) +>>> Factor productivity (Total, labour share and capital share) +>>> Labour share in total factor productivity: total economy (ZVGDE) +>>> Total factor productivity: total economy (ZVGDF) +>>> Capital share in total factor productivity: total economy (ZVGDK) +>> Labour-capital substitution +>>> Labour-capital substitution: total economy (ZKNDE) +>> Capital-labour substitution +>>> Capital-labour substitution: total economy (ZEKND) +>> Marginal efficiency of capital +>>> Marginal efficiency of capital; total economy (AKGDV) +>> Net returns on net capital stock +>>> Net returns on net capital stock; total economy (APNDK) +> Exports and imports of goods and services +>> Exports of goods and services +>>> Exports of goods and services at constant prices (OXGS) +>>> Price deflator exports of goods and services (PXGS) +>>> Exports of goods and services at current prices (National accounts) (UXGS) +>>> Export markets: Export weighted imports ; Goods and services at constant prices: 23 industrial markets (EU-14, CHE NOR USA CAN JAP AUS MEX NZL and TUR) (VMGSW) +>>> Market performance of exports of goods and services on export weighted imports of goods and services ; 23 industrial markets (EU-14, CHE NOR USA CAN JAP AUS MEX NZL and TUR) (VXGSP) +>>> Exports of goods and services at constant prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OXGSQ) +>>> Price deflator exports of goods and services ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PXGSQ) +>>> Exports of goods and services at constant prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OXGSQ) +>>> Price deflator exports of goods and services ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PXGSQ) +>> Imports of goods and services +>>> Imports of goods and services at constant prices (OMGS) +>>> Price deflator imports of goods and services (PMGS) +>>> Imports of goods and services at current prices (National accounts) (UMGS) +>>> Terms of trade (goods and services) +>>> Terms of trade goods and services (National accounts; 1995 = 100) (APGS) +>>> Impact of terms of trade goods and services on real income (APTA) +>> Exports of goods +>>> Exports of goods at constant prices (OXGN) +>>> Price deflator exports of goods (PXGN) +>>> Exports of goods at current prices (National accounts) (UXGN) +>> Imports of goods +>>> Imports of goods at constant prices (OMGN) +>>> Price deflator imports of goods (PMGN) +>>> Imports of goods at current prices (National accounts) (UMGN) +>>> Terms of trade (goods) +>>> Terms of trade goods (National accounts) (APGN) +> Balances with the rest of the world +>> Net exports of goods and services +>>> Net exports of goods and services at current prices (National accounts) (UBGS) +>> Net exports of goods +>>> Net exports of goods at current prices (National accounts) (UBGN) +>> Balance of primary income from the rest of the world +>>> Net factor income from the Rest of the world (National accounts) (UBRA) +>> Balance of current transfers with the rest of the world +>>> Net current transfers from the rest of the world (National accounts) (UBTA) +>> Balance on current transactions with the rest of the world +>>> Balance on current transactions with the rest of the world (National accounts) (UBCA) +>> Balance of capital transfers with the rest of the world +>>> Net capital transactions with the rest of the world (National accounts) (UBKA) +>> Net lending (+) or net borrowing (-) of the nation +>>> Net lending (+) or net borrowing (-); total economy (UBLA) +> Foreign trade at current prices +>> Exports goods (FOB) total +>>> Total exports of goods (FOB); Foreign trade statistics (DXGT) +>>> Total exports of goods excluding energy products; Foreign trade statistics (DXGT3) +>> Exports goods (FOB) intra EU +>>> Intra EU exports of goods; Foreign trade statistics (DXGI) +>>> Intra EU exports of goods excluding energy products; Foreign trade statistics (DXGI3) +>> Exports goods (FOB) extra EU +>>> Extra EU exports of goods; Foreign trade statistics (DXGE) +>>> Extra EU exports of goods excluding energy products; Foreign trade statistics (DXGE3) +>> Imports goods (CIF) total +>>> Total imports of goods (CIF) ; Foreign trade statistics (DMGT) +>>> Total imports of goods excluding energy products; Foreign trade statistics (DMGT3) +>> Imports goods (CIF) intra EU +>>> Intra EU imports of goods (CIF); Foreign trade statistics (DMGI) +>>> Intra EU imports of goods excluding energy products; Foreign trade statistics (DMGI3) +>> Imports goods (CIF) extra EU +>>> Extra EU imports of goods; Foreign trade statistics (DMGE) +>>> Extra EU imports of goods excluding energy products; Foreign trade statistics (DMGE3) +>> Foreign trade shares in world trade +>>> Average share of imports and exports of goods in world trade excluding intra EU trade; Foreign trade statistics (AAGE) +>>> Average share of imports and exports of goods in world trade including intra EU trade; Foreign trade statistics (AAGT) +>>> Share of imports of goods in world imports excluding intra EU imports ; Foreign trade statistics (AMGE) +>>> Share of imports of goods in world imports including intra EU imports-; Foreign trade statistics (AMGT) +>>> Share of exports of goods in world exports excluding intra EU exports; Foreign trade statistics (AXGE) +>>> Share of exports of goods in world exports including intra EU exports; Foreign trade statistics (AXGT) +>>> National accounts by branch of activity (Part I) +>> Occupied population +>>> Employment, full-time equivalents agriculture, forestry and fishery products (National accounts) (FET1) +>>> Employment, full-time equivalents; industry excluding building and construction (National accounts) (FET2) +>>> Employment, full-time equivalents; building and construction (National accounts) (FET4) +>>> Employment, full-time equivalents; services (National accounts) (FET5) +>>> Employment, full-time equivalents; manufacturing industry (National accounts) (FETM) +>>> Employment, persons; agriculture, forestry and fishery products (National accounts) (NET1) +>>> Employment, persons; industry excluding building and construction (National accounts) (NET2) +>>> Employment, persons; building and construction (National accounts) (NET4) +>>> Employment, persons; services (National accounts) (NET5) +>>> Employment, persons; manufacturing industry (National accounts) (NETM) +>> Wage and salary earners +>>> Employees, full-time equivalents; agriculture, forestry and fishery products (National accounts) (FWT1) +>>> Employees, full-time equivalents; industry excluding building and construction (National accounts) (FWT2) +>>> Employees, full-time equivalents; building and construction (National accounts) (FWT4) +>>> Employees, full-time equivalents; services (National accounts) (FWT5) +>>> Employees, full-time equivalents; manufacturing industry (National accounts) (FWTM) +>>> Employees, persons: agriculture, forestry and fishery products (National accounts) (NWT1) +>>> Employees, persons; industry excluding building and construction (National accounts) (NWT2) +>>> Employees, persons; building and construction (National accounts) (NWT4) +>>> Employees, persons; services (National accounts) (NWT5) +>>> Employees, persons; manufacturing industry (National accounts) (NWTM) +>> Gross value added at current prices +>>> Gross value added at current prices; total of branches (UVG0) +>>> Gross value added at current prices; agriculture, forestry and fishery products (UVG1) +>>> Gross value added at current prices; industry excluding building and construction (UVG2) +>>> Gross value added at current prices; building and construction (UVG4) +>>> Gross value added at current prices; services (UVG5) +>>> Gross value added at current prices; manufacturing industry (UVGM) +>> Gross value added at current prices per person employed +>>> Gross value added at current prices per person employed; agriculture, forestry and fishery products (HVG1E) +>>> Gross value added at current prices per person employed; industry excluding building and construction (HVG2E) +>>> Gross value added at current prices per person employed; building and construction (HVG4E) +>>> Gross value added at current prices per person employed; services (HVG5E) +>>> Gross value added at current prices per person employed; manufacturing industry (HVGME) +>> Gross value added at current prices per employee +>>> Gross value added at current prices per employee; manufacturing industry (HVGMW) +>> Gross value added at constant prices +>>> Gross value added at constant prices; total of branches (OVG0) +>>> Gross value added at constant prices; agriculture, forestry and fishery products (OVG1) +>>> Gross value added at constant prices industry excluding building and construction (OVG2) +>>> Gross value added at constant prices; building and construction (OVG4) +>>> Gross value added at constant prices; services (OVG5) +>>> Gross value added at constant prices; manufacturing industry (OVGM) +>>> National accounts by branch of activity (Part II) +>> Gross value added at constant prices per person employed +>>> Real unit labour costs; manufacturing industry (QLCM) +>>> Gross value added at constant prices per person employed; agriculture, forestry and fishery products (RVG1E) +>>> Gross value added at constant prices per person employed; industry excluding building and construction (RVG2E) +>>> Gross value added at constant prices per person employed; building and construction (RVG4E) +>>> Gross value added at constant prices per person employed; services (RVG5E) +>>> Gross value added at constant prices per person employed; manufacturing industry (RVGME) +>> Gross value added at constant prices per employee +>>> Gross value added at constant prices per employee; manufacturing industry (RVGMW) +>> Price deflator gross value added +>>> Price deflator gross value added; agriculture, forestry and fishery products (PVG1) +>>> Price deflator gross value added; industry excluding building and construction (PVG2) +>>> Price deflator gross value added; building and construction (PVG4) +>>> Price deflator gross value added; services (PVG5) +>>> Price deflator gross value added; manufacturing industry (PVGM) +>> Industrial production +>>> Industrial production, construction excluded (VPRI) +>> Compensation of employees +>>> Compensation of employees; agriculture, forestry and fishery products (UWC1) +>>> Compensation of employees; industry excluding building and construction (UWC2) +>>> Compensation of employees; building and construction (UWC4) +>>> Compensation of employees; services (UWC5) +>>> Compensation of employees; manufacturing industry (UWCM) +>> Nominal compensation per employee +>>> Nominal compensation per employee; agriculture, forestry and fishery products (HWC1W) +>>> Nominal compensation per employee; industry excluding building and construction (HWC2W) +>>> Nominal compensation per employee; building and construction (HWC4W) +>>> Nominal compensation per employee; services (HWC5W) +>>> Nominal compensation per employee; manufacturing industry (HWCMW) +>>> Adjusted wage share (manufacturing) +>>> Adjusted wage share; manufacturing industry (ALCM) +>> Unit labour and wage costs +>>> Nominal unit wage costs; agriculture, forestry and fishery products (1995 = 100) (PWC1) +>>> Nominal unit wage costs; industry excluding building and construction (1995 = 100) (PWC2) +>>> Nominal unit wage costs; building and construction (1995 = 100) (PWC4) +>>> Nominal unit wage costs; services (1995 = 100) (PWC5) +>>> Nominal unit wage costs; manufacturing industry (1995 = 100) (PWCM) +>>> Nominal unit labour costs; manufacturing industry (1995) (PLCM) +> Monetary variables +>> Nominal exchange rates +>>> ECU-EUR exchange rates (XNE) +>> Nominal exchange rates +>>> Conversion rates between euro and former euro-zone national currencies (XNEF) +>> Nominal effective exchange rates +>>> Nominal effective exchange rates; Performance relative to the rest of 35 industrial countries; double export weights (XUNNQ) +>> Effective exchange rates +>>> Real effective exchange rates ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (XUNRQ) +>> GDP purchasing power parities +>>> GDP purchasing power parities (KNP) +>> Interest rates +>>> Nominal long-term interest rates (ILN) +>>> Real long-term interest rates, deflator private consumption (ILRC) +>>> Real long-term interest rates, deflator GDP (ILRV) +>>> Nominal short-term interest rates (ISN) +>>> Real short-term interest rates, deflator private consumption (ISRC) +>>> Real short-term interest rates, deflator GDP (ISRV) +>>> Yield curve (IYN) +> Non-financial and financial corporations (S11 + S12) +>> Revenue +>>> Other subsidies on production; corporations (UYVC) +>>> Net current transfers received; corporations (UCTRC) +>>> Net property income; corporations (UYNC) +>> Expenditure +>>> Adjustment for the change in net equity of households in pension funds; corporations (UEHC) +>>> Gross capital formation; corporations (UITC) +>>> Other capital expenditure, net; corporations (UKOC) +>>> Other taxes on production; corporations (UTVC) +>>> Current taxes on income and wealth; corporations (UTYC) +>>> Compensation of employees-; corporations (UWCC) +>> Balances +>>> Gross value added at basic prices; corporations (UGVAC) +>>> Net lending (+) or net borrowing (-); corporations (UBLC) +>>> Gross saving; corporations (USGC) +>>> Net saving; corporations (USNC) +>>> Gross disposable income; corporations (UVGC) +>>> Net disposable income; corporations (UVNC) +>>> Gross operating surplus; corporations (UOGC) +>>> Gross balance of primary income; corporations (UBGC) +>>> Net balance of primary income; corporations (UBNC) +> Households and NPISH (S14 + S15) +>> Revenue +>>> Compensation of employees; households and NPISH (UWCH) +>>> Gross wages and salaries; households and NPISH (UWSH) +>>> Non-labour income; households and NPISH (UYOH) +>>> Net property income; households and NPISH (UYNH) +>>> Current transfers received; households and NPISH (UCTRH) +>> Expenditure +>>> Net lending (+) or net borrowing (-); households and NPISH (UBLH) +>>> Other capital expenditure, net; households and NPISH (UKOH) +>>> Gross saving; households and NPISH (USGH) +>>> Current taxes on income and wealth; households and NPISH (UTYH) +>>> Current transfers paid; households and NPISH (UCTPH) +>> Balances +>>> Saving rate, gross; households and NPISH (ASGH) +>>> Real gross disposable income, deflator private consumption; households and NPISH (OVGH) +>>> Net saving; households (USNH) +>>> Gross operating surplus and mixed income; households and NPISH (UOGH) +>>> Gross disposable income; households and NPISH (UVGH) +>>> Final consumption expenditure; households and NPISH (UCPH0) +>>> Gross capital formation; households and NPISH (UITH) +> General government (S13) +>> Revenue +>>> Capital transfers received; general government; ESA 1995 (UKTTG) +>>> Total revenue; general government; ESA 1995 (URTG) +>>> Total tax burden excluding imputed social security contributions; total economy; ESA 1995 (UTAT) +>>> Total tax burden including imputed social security contributions; total economy; ESA 1995 (UTTT) +>>> Taxes linked to imports and production (indirect taxes); general government; ESA 1995 (UTVG) +>> Taxes on production and imports comprise: +>>> Current taxes on income and wealth (direct taxes); general government; ESA 1995 (UTYG) +>>> Other current revenue; general government; ESA 1995 (UROG) +>>> Actual social contributions received; general government; ESA 1995 (UTAG) +>>> Imputed social contributions; general government; ESA 1995 (UTIG) +>>> Current tax burden; total economy; ESA 1995 (UTCT) +>>> Capital taxes; general government; ESA 1995 (UTKG) +>>> Social contributions received; general government; ESA 1995 (UTSG) +>>> Total current revenue; general government; ESA 1995 (URCG) +>> Expenditure +>>> Implicit interest rate; general government; -ESA 1995 (AYIGD) +>>> Final consumption expenditure of general government; ESA 1995 (UCTG0) +>>> Interest; general government; ESA 1995 (UYIG) +>>> Interest- including flows on swaps and FRAs (Forward Rate Agreements); general government; Excessive deficit procedure (UYIGE) +>>> Social transfers other than in kind; general government-; ESA 1995 (UYTGH) +>>> Subsidies; general government; ESA 1995 (UYVG) +>>> Real total expenditure of general government, deflator GDP; ESA 1995 (OUTG) +>>> Real total expenditure excluding interest of general government, deflator GDP; ESA 1995 (OUTGI) +>>> Collective consumption expenditure; ESA 1995 (UCCG0) +>>> Total current expenditure excluding interest; general government; ESA 1995 (UUCGI) +>>> Total expenditure excluding interest; general government; ESA 1995 (UUTGI) +>>> Social transfers in kind; ESA 1995 (UCIG0) +>>> Compensation of employees; general government; ESA 1995 (UWCG) +>>> Other current expenditure; general government; ESA 1995 (UUOG) +>>> Total current expenditure; general government; ESA 1995 (UUCG) +>>> Total current expenditure; general government; Excessive deficit procedure (UUCGE) +>>> Capital transfers paid; general government; ESA 1995 (UKTGT) +>>> Gross fixed capital formation; general government; ESA 1995 (UIGG0) +>>> Total expenditure; general government; ESA 1995 (UUTG) +>>> Total expenditure; general government; Excessive deficit procedure (UUTGE) +>>> Other capital expenditure, including capital transfers; general government; ESA 1995 (UKOG) +>> Balances +>>> Net lending (+) or net borrowing (-); general government; ESA 1995 (UBLG) +>>> Net lending (+) or net borrowing (-); general government; Excessive deficit procedure (UBLGE) +>>> Net lending (+) or net borrowing (-) excluding interest; general government; ESA 1995 (UBLGI) +>>> Net lending (+) or net borrowing (-) excluding interest: general government; Excessive deficit procedure (UBLGIE) +>>> Gross saving; general government; ESA 1995 (USGG) +>>> Gross saving; general government; Excessive deficit procedure (USGGE) +>>> Gross disposable income; general government; ESA 1995 (UVGG) +>>> Net disposable income; general government; ESA 1995 (UVNG) +>>> Net lending (+) or net borrowing (-) excluding gross fixed capital formation; general government; ESA 1995 (UBLGG) +>>> Public finance (other variables) +>> Cyclical adjustment of public finance variables based on TREND GDP +>>> Cyclically adjusted net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGA) +>>> Net lending (+) or net borrowing (-) excluding interest of general government adjusted for the cyclical component; ESA 1995 (UBLGB) +>>> Cyclical component of net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGC) +>>> Cyclically adjusted total revenue of general government; ESA 1995 (URTGA) +>>> Cyclical component of revenue of general government; ESA 1995 (UTCGC) +>>> Cyclical component of expenditure of general government (UUCGC) +>>> Cyclically adjusted total expenditure of general government; ESA 1995 (UUTGA) +>>> Total expenditure excluding interest of general government adjusted for the cyclical component; ESA 1995 (UUTGB) +>> Cyclical adjustment of public finance variables based on POTENTIAL GDP +>>> Cyclically adjusted net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGAP) +>>> Net lending (+) or net borrowing (-) excluding interest of general government adjusted for the cyclical component; ESA 1995 (UBLGBP) +>>> Cyclical component of net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGCP) +>>> Cyclically adjusted total revenue of general government; ESA 1995 (URTGAP) +>>> Cyclical component of revenue of general government; ESA 1995 (UTCGCP) +>>> Cyclical component of expenditure of general government (UUCGCP) +>>> Cyclically adjusted total expenditure of general government; ESA 1995 (UUTGAP) +>>> Total expenditure excluding interest of general government adjusted for the cyclical component; ESA 1995 (UUTGBP) +>> Gross public debt +>>> Snow ball effect on general government consolidated gross debt; ESA 1995 (ADGGI) +>>> Impact of the nominal increase of GDP on general government consolidated gross debt; ESA 1995 (ADGGU) +>>> Stock-flow adjustment on general government consolidated gross debt; ESA 1995 (UDGGS) +>>> General government consolidated gross debt; ESA 1995 (UDGG) +>>> General government consolidated gross debt ; EU-Member States: ESA 95 and former definition (linked series) (UDGGL) +>>> General government consolidated gross debt; ESA 1995 (UDGGR) \ No newline at end of file diff --git a/larray_editor/editor.py b/larray_editor/editor.py index d26f7c0..abcf11d 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -8,6 +8,8 @@ from contextlib import redirect_stdout from pathlib import Path from typing import Union +from larray_eurostat import freq_eurostat_editor, eurostat_get +from larray_ameco.get_ameco import reshape_and_dump_ameco_dataset # Python3.8 switched from a Selector to a Proactor based event loop for asyncio but they do not offer the same @@ -37,14 +39,16 @@ get_versions, get_documentation_url, urls, RecentlyUsedList) from larray_editor.arraywidget import ArrayEditorWidget from larray_editor.commands import EditSessionArrayCommand, EditCurrentArrayCommand -from larray_editor.treemodel import SimpleTreeNode, SimpleLazyTreeModel +from larray_editor.treemodel import SimpleTreeNode, SimpleLazyTreeModel, Ameco_parse_tree_structure -from qtpy.QtCore import Qt, QUrl, QSettings + +from qtpy.QtCore import Qt, QUrl, QSettings, QFileInfo, QThread, Signal from qtpy.QtGui import QDesktopServices, QKeySequence from qtpy.QtWidgets import (QMainWindow, QWidget, QListWidget, QListWidgetItem, QSplitter, QFileDialog, QPushButton, QDialogButtonBox, QShortcut, QVBoxLayout, QHBoxLayout, QGridLayout, QLineEdit, QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton, - QTreeView, QTextEdit, QMenu, QAction) + QTreeView, QTextEdit, QMenu, QAction, QTreeWidget, QTreeWidgetItem) + try: from qtpy.QtWidgets import QUndoStack @@ -88,6 +92,20 @@ DISPLAY_IN_GRID = (la.Array, np.ndarray) +def multi_search_df(df, text): + terms = text.split() + mask = None + + # Takes 'boolean intersection' with previous search result(s) + for term in terms: + term_mask = (df['Code'].str.contains(term, case=False, na=False)) | (df['Title'].str.contains(term, case=False, na=False)) + if mask is None: + mask = term_mask + else: + mask &= term_mask + return df[mask] + + def num_leading_spaces(s): i = 0 while s[i] == ' ': @@ -119,52 +137,6 @@ def indented_df_to_treenode(df, indent=4, indented_col=0, colnames=None, header= return root -# Recursively traverse tree and extract the data *only* of leaf nodes -def traverse_tree(node, rows): - # If the node has no children, append its data to rows - if not node.children: - rows.append(node.data) - # If the node has children, recursively call the function for each child - for child in node.children: - traverse_tree(child, rows) - - -# Put all the leaf nodes of tree into a dataframe structure -def tree_to_dataframe(root): - rows = [] - traverse_tree(root, rows) - return pd.DataFrame(rows, columns=root.data) - - -# Extract frequencies from given dataset and subset accordingly -def freq_eurostat(freqs, la_data): - # If "TIME_PERIOD" exists in la_data's axes names, rename it to "time" - if "TIME_PERIOD" in la_data.axes.names: - freq_data = la_data.rename(TIME_PERIOD='time') - - a_time = [] - - for freq in freqs: - # str() because larray labels are not always strings, might also return ints - if freq == 'A': - a_time += [t for t in freq_data.time.labels if '-' not in str(t)] - elif freq == 'Q': - a_time += [t for t in freq_data.time.labels if 'Q' in str(t)] - elif freq == 'M': - a_time += [t for t in freq_data.time.labels if '-' in t and 'Q' not in str(t)] - - # Maintain order and use set for non-duplicates - a_time = sorted(set(a_time)) - - if len(freqs) == 1 and freq[0] in ['A', 'Q', 'M']: - freq_value = str(freqs[0]) - else: - freq_value = freqs - - # Return with row and colum-wise subsetting - return freq_data[freq_value, a_time] - - class FrequencyFilterDialog(QDialog): def __init__(self, available_labels, parent=None): @@ -198,7 +170,425 @@ def get_selected_frequencies(self): if checkbox.isChecked(): freqs.append(label) return freqs - + + + +class AmecoDownloadThread(QThread): + # Needs to run in Seperate Thread to avoid blocking the UI + finishedSignal = Signal() + errorSignal = Signal(str) + + def run(self): + print("Inside the download thread...") + try: + reshape_and_dump_ameco_dataset() + print("Finished downloading in the thread.") + self.finishedSignal.emit() + except Exception as e: + errorMsg = f"An unexpected error occurred: {str(e)}" + self.errorSignal.emit(errorMsg) + + + +class AmecoBrowserDialog(QDialog): + def __init__(self, ameco_table_of_contents, parent=None): + super(AmecoBrowserDialog, self).__init__(parent) + + # Basic UI properties + self.title = 'AMECO Database' + self.ameco_table_of_contents = ameco_table_of_contents + + # Initialize UI elements + self.init_ui() + + + def init_ui(self): + self.init_widgets() + self.set_properties() + self.connect_signals_slots() + self.configure_layout() + self.populate_tree(self.ameco_table_of_contents) + self.show() + + + def init_widgets(self): + self.tree = QTreeWidget() + self.search_input = QLineEdit(self) + self.search_results_list = QListWidget() + self.refresh_button = QPushButton("Last Updated: ", self) + self.update_text_for_last_updated_button() # Means here: put initial text for 'Last Update' button + self.downloadThread = AmecoDownloadThread() + + + def set_properties(self): + self.resize(750, 600) + self.setWindowTitle(self.title) + self.tree.setHeaderHidden(True) + self.search_input.setPlaceholderText("Search...") + self.search_results_list.hide() + + + def connect_signals_slots(self): + # Connect for search functionalities + self.search_input.textChanged.connect(self.handle_search) + self.search_results_list.itemClicked.connect(self.handle_search_item_click) + + # Connect for refresh button and finished download + self.refresh_button.clicked.connect(self.start_download) + self.downloadThread.finishedSignal.connect(self.update_text_for_last_updated_button) + + # Connect for treeitem click + self.tree.itemDoubleClicked.connect(self.on_tree_item_clicked) + + + def configure_layout(self): + # Search layout configuration + search_layout = QHBoxLayout() + search_layout.addWidget(self.search_input) + search_layout.addWidget(self.refresh_button) + + # Main layout configuration + layout = QVBoxLayout() + layout.addLayout(search_layout) + layout.addWidget(self.search_results_list) + layout.addWidget(self.tree) + self.setLayout(layout) + + + def populate_tree(self, ameco_table_of_contents): + root_tree_node = Ameco_parse_tree_structure(ameco_table_of_contents.split('\n')) + for child in root_tree_node.children: + self.ameco_add_nodes(self.tree, child) + + + def start_download(self): + if not self.downloadThread.isRunning(): + # Show a popup box to inform the user that the download has started. + msgBox = QMessageBox() + msgBox.setText("Download started. AMECO datasets are published once per year.") + msgBox.setIcon(QMessageBox.Information) + msgBox.setStandardButtons(QMessageBox.Ok) + msgBox.exec_() + + self.downloadThread.start() + self.downloadThread.setPriority(QThread.HighPriority) + + + def update_text_for_last_updated_button(self): + print("Updating 'Last Updated' after download (or init)...") + + # Get the last modified datetime + file_path = '__array_cache__/data.h5' + info = QFileInfo(file_path) + last_modified = info.lastModified().toString("dd-MM-yyyy HH:mm:ss") + + # Update the button's label + self.refresh_button.setText(f"Last Updated: {last_modified}") + + + def ameco_tree_to_dataframe(self, ameco_table_of_contents): + rows = [] + lines = ameco_table_of_contents.split('\n') + for line in lines: + # Match title and code using regex + match = re.match(r'^(.*)\s+\(([^)]+)\)$', line) + if match: + title, code = match.groups() + # Remove prefixes >, >>, and >>> + title = title.replace('>', '').strip() + rows.append({'Title': title, 'Code': code.strip()}) + + df = pd.DataFrame(rows) + return df + + + def ameco_add_nodes(self, qt_parent, tree_node): + qt_node = QTreeWidgetItem(qt_parent, [tree_node.data]) + for child in tree_node.children: + self.ameco_add_nodes(qt_node, child) + + + def handle_search(self, text): + self.df = self.ameco_tree_to_dataframe(self.ameco_table_of_contents) + + if text: + self.tree.hide() + self.search_results_list.clear() + + filtered_df = multi_search_df(self.df, text) + filtered_df = filtered_df.drop_duplicates(subset='Code') + results = [f"{row['Title']} ({row['Code']})" for _, row in filtered_df.iterrows()] + + self.search_results_list.addItems(results) + self.search_results_list.show() + + else: # if search field is empty + self.tree.show() + self.search_results_list.hide() + + + def handle_search_item_click(self, item): + # Assume the substring '(code)' is always the last thing in parentheses + matches = re.findall(r'\(([^)]+)\)', item.text()) + + if matches: + code = matches[-1] + popup = AmecoSettingsDialog(code, self) + popup.exec_() + + + def on_tree_item_clicked(self, item): + # Extracts the code of item clicked and opens next Popup dialog. + if item.childCount() == 0: + import re + match = re.findall(r'\(([^)]+)\)', item.text(0)) # search last substring '(code)' + if match: + var_name = match[-1] + else: + var_name = "" + + # print(f"the extracted code of clicked item is: {var_name}") + popup = AmecoSettingsDialog(var_name, self) + popup.exec_() + + + def keyPressEvent(self, event): + if event.key() in [Qt.Key_Return, Qt.Key_Enter] and self.tree.hasFocus(): + item = self.tree.currentItem() + if item and item.childCount() == 0: + matches = re.findall(r'\(([^)]+)\)', item.text(0)) # search last substring '(code)' + if matches: + code = matches[-1] + popup = AmecoSettingsDialog(code, self) + popup.exec_() + + + +class AmecoSettingsDialog(QDialog): + def __init__(self, var_name, parent=None): + super().__init__(parent) + self.resize(500, 150) + self.setWindowTitle("Settings") + self.load_data(var_name) + self.init_ui(var_name) + + + def load_data(self, var_name): + self.ameco = la.read_hdf("__array_cache__/data.h5", "ameco0") + all_numerical_codes = [str(self.ameco.axes[0][i]) for i in self.ameco.axes[0]] # adding str() is necessary! + + # Given a *particular* var name, most numerical codes deliver NaN arrays. Consider only relevant numerical codes. + relevant_numerical_codes_for_var = [] + for id in all_numerical_codes: + is_all_nan = la.isnan(self.ameco[id, var_name]).all() + + if not is_all_nan: + relevant_numerical_codes_for_var.append([int(id_part) for id_part in id.split("-")]) # "1-2-3-4" [str] -> [1,2,3,4] + + # Put only relevant codes in dropdowns (= i.e. only need subset of the total 27 combo possibilities) + self.choice_list = relevant_numerical_codes_for_var + print(f"All possible choices: {self.choice_list}") # temp use for debugging, in case some codes are not listed in official document + + + def init_ui(self, var_name): + layout = QVBoxLayout() + self.initialize_content_dropdowns() + + self.drop1, self.drop2, self.drop3, self.drop4 = QComboBox(), QComboBox(), QComboBox(), QComboBox() + self.populate_dropdowns() + self.setup_dropdown_grid(layout) + self.setup_action_elements(layout, var_name) + self.setLayout(layout) + + + def initialize_content_dropdowns(self): + # Mapping dictionaries for dropdowns (codes -> description). See AMECO documentation for more info. + self.drop1_options = { + 1: 'Levels (and moving arithmetic mean for time periods)', + 2: 'Levels (and moving geometric mean for time periods)', + 3: 'Index numbers (and moving arithmetic mean for time periods)', + 4: 'Index numbers (and moving geometric mean for time periods)', + 5: 'Annual percentage changes (and moving arithmetic mean for time periods)', + 6: 'Annual percentage changes (and moving geometric mean for time periods)', + 7: 'Absolute value of annual percentage changes (and moving arithmetic mean for time periods)', + 8: 'Moving percentage changes', + 9: 'Annual changes (and moving arithmetic mean for time periods)', + 10: 'Absolute value of annual changes (and moving arithmetic mean for time periods)', + 11: 'Moving changes', + } + + self.drop2_options = { + 0: 'Standard aggregations (data converted to a common currency and summed)', + 1: 'Weighted mean of t/t-1 national ratios, weights current prices in ECU/EUR', + 2: 'Weighted mean of t/t-1 national ratios, weights current prices in PPS', + } + + self.drop3_options = { + 0: 'Original units (e.g. national currency, persons, etc.)', + 99: 'ECU/EUR', + 212: 'PPS (purchasing power standards)', + 300: 'Final demand', + 310: 'Gross domestic product at market prices', + 311: 'Net domestic product at market prices', + 312: 'Gross domestic product at current factor cost', + 313: 'Net domestic product at current factor cost', + 315: 'Trend gross domestic product at market prices', + 316: 'Potential gross domestic product at market prices', + 318: 'Total gross value added', + 319: 'Gross domestic product at market prices (excessive deficit procedure)', + 320: 'Gross national product at market prices', + 321: 'National income at market prices', + 322: 'Gross national disposable income at market prices', + 323: 'National disposable income at market prices', + 338: 'Gross value added at market prices; manufacturing industry', + 380: 'Current revenue general government', + 390: 'Total expenditure general government', + 391: 'Current expenditure general government', + 410: 'Total population, demographic statistics', + 411: 'Population, 15 to 64 years', + 412: 'Total labour Force', + 413: 'Civilian labour force', + 414: 'Population 15 to 74 years', + 420: 'Total population (national accounts)', + } + + self.drop4_options = { + 0: 'value for the reporting country', # default + 215: 'former EU-15', + 315: 'former EU-15', + 327: 'former EU-27', # added extra: not in official docs? but sometimes used, cf. 'HVGDPR' + 328: 'former EU-28', + 415: 'former EU-15 (bis)', # in documentation double code (315), but then issue with reverse dictionary map, so change text slightly + 424: '24 industrial countries: former EU-15 & CH NR TR US CA MX JP AU NZ', + 437: '37 industrial countries: EU-27 & CH NR TR UK US CA MX JP AU NZ', + } + + + def populate_dropdowns(self): + # Some sets to keep track of adding procedure (i.e. useful to avoid double adds) + added_drop1, added_drop2, added_drop3, added_drop4 = set(), set(), set(), set() + + for item in self.choice_list: + # extract fourfold from iterator 'item' (which is a list of 4 values, e.g. [1,0,0,0]) + drop1_val, drop2_val, drop3_val, drop4_val = item + + if drop1_val in self.drop1_options and drop1_val not in added_drop1: + self.drop1.addItem(self.drop1_options[drop1_val]) + added_drop1.add(drop1_val) + + if drop2_val in self.drop2_options and drop2_val not in added_drop2: + self.drop2.addItem(self.drop2_options[drop2_val]) + added_drop2.add(drop2_val) + + if drop3_val in self.drop3_options and drop3_val not in added_drop3: + self.drop3.addItem(self.drop3_options[drop3_val]) + added_drop3.add(drop3_val) + + if drop4_val in self.drop4_options and drop4_val not in added_drop4: + self.drop4.addItem(self.drop4_options[drop4_val]) + added_drop4.add(drop4_val) + + + def setup_dropdown_grid(self, layout): + grid = QGridLayout() + + labels = [ + ("TRN: Transformations over time", self.drop1), + ("AGG: Aggregation modes", self.drop2), + ("UNIT: Unit codes", self.drop3), + ("REF: Codes for relative performance", self.drop4) + ] + + # Create grid layout for labels and dropdowns + grid = QGridLayout() + for index, (label_text, dropdown) in enumerate(labels): + label = QLabel(label_text) + + # Setting a fixed width ensures all labels have the same width. + # Add some extra whitespaces at end -- for beter alignment. + label.setFixedWidth(label.fontMetrics().width("REF: Codes for relative performance ")) + + grid.addWidget(label, index, 0) + grid.addWidget(dropdown, index, 1) + + layout.addLayout(grid) + + + def setup_action_elements(self, layout, var_name): + self.submit_button = QPushButton("Load Data", self) + self.submit_button.clicked.connect(lambda: self.ameco_final_submit_clicked(var_name)) + # self.submit_button.clicked.connect(self.ameco_final_submit_clicked(var_name)) # check: why only lambda works?? + + # Invalid choice label + self.invalid_label = QLabel("Invalid selection. Please choose a valid combination from the dropdown menus.", self) + self.invalid_label.setStyleSheet("color: red;") + self.invalid_label.hide() # Hide it initially + + layout.addWidget(self.invalid_label) + layout.setAlignment(self.invalid_label, Qt.AlignCenter) + layout.addWidget(self.submit_button) + + # Connect dropdowns to verification function + self.drop1.currentIndexChanged.connect(self.verification_check) + self.drop2.currentIndexChanged.connect(self.verification_check) + self.drop3.currentIndexChanged.connect(self.verification_check) + self.drop4.currentIndexChanged.connect(self.verification_check) + + # Set layout + self.setLayout(layout) + + + def verification_check(self, silent=False): + # At this stage, we have a 'var_name' and 'dropdown selection' provided by user. + # Revert now back to numerical codes so we can subset/load correct larray data. + + # Reverse mapping for dropdowns. Go back from text in dropdowns to numerical codes. + def create_reverse_mapping(original_mapping): + return {v: k for k, v in original_mapping.items()} + + # Apply reverse mappings + rev_drop1_options = create_reverse_mapping(self.drop1_options) + rev_drop2_options = create_reverse_mapping(self.drop2_options) + rev_drop3_options = create_reverse_mapping(self.drop3_options) + rev_drop4_options = create_reverse_mapping(self.drop4_options) + + # Get Selected Options + self.sel1 = rev_drop1_options[self.drop1.currentText()] + self.sel2 = rev_drop2_options[self.drop2.currentText()] + self.sel3 = rev_drop3_options[self.drop3.currentText()] + self.sel4 = rev_drop4_options[self.drop4.currentText()] + + # Merge together 4 dropdown selections to single code (e.g. '1-1-0-0') for subsetting larray + self.numo_code = "-".join([str(self.sel1), str(self.sel2), str(self.sel3), str(self.sel4)]) + + # Extra check if valid choice was made in dropdown combinations. + if [self.sel1, self.sel2, self.sel3, self.sel4] in self.choice_list: + print(f"valid choice: {self.numo_code}") + self.submit_button.show() + self.invalid_label.hide() + return True + else: + print(f"invalid choice: {self.numo_code}") + self.submit_button.hide() + self.invalid_label.show() + if not silent: + QMessageBox.information(self, f"{self.numo_code} is a bad combination.", f"Valid choices are: {self.choice_list}", QMessageBox.Ok) + return False + + def ameco_final_submit_clicked(self, var_name): + if self.verification_check(silent=True): + ameco_subset = self.ameco[self.numo_code, var_name] + + editor = self.parent().parent() + new_data = editor.data.copy() # make a copy of dataset-dictionary before starting to add new datasets + new_data[var_name] = ameco_subset # add dataset for LArrayEditor update + editor.kernel.shell.user_ns[var_name] = ameco_subset # add dataset to console namespace + editor.view_expr(ameco_subset, expr=var_name) + editor.update_mapping(new_data) + + self.accept() + + class EurostatBrowserDialog(QDialog): def __init__(self, index, parent=None): @@ -218,7 +608,7 @@ def __init__(self, index, parent=None): # Simple*Lazy*TreeModel, but it is fast enough for now. *If* it ever becomes a problem, # we could make this lazy pretty easily (see treemodel.LazyDictTreeNode for an example). root = indented_df_to_treenode(index) - self.df = tree_to_dataframe(root) + self.df = self.eurostat_tree_to_dataframe(root) # Create the tree view UI in Dialog model = SimpleLazyTreeModel(root) @@ -247,7 +637,7 @@ def __init__(self, index, parent=None): self.advanced_button.clicked.connect(self.showAdvancedPopup) # General settings: resize + title - self.resize(450, 600) + self.resize(950, 600) self.setWindowTitle("Select dataset") # Add widgets to layout @@ -259,6 +649,18 @@ def __init__(self, index, parent=None): self.setLayout(layout) + def eurostat_tree_to_dataframe(self, root): + # Recursively traverse tree and extract data *only* of leaf nodes (into dataframe) + def traverse_tree(node, rows): + if not node.children: # If the node has no children, append its data to rows + rows.append(node.data) + for child in node.children: # If the node has children, recursively call the function for each child + traverse_tree(child, rows) + rows = [] + traverse_tree(root, rows) + return pd.DataFrame(rows, columns=root.data) + + def show_context_menu(self, position): menu = QMenu(self) @@ -276,11 +678,10 @@ def show_context_menu(self, position): def add_to_list(self): - from larray_eurostat import eurostat_get index = self.tree.currentIndex() node = index.internalPointer() - if not node.children: # should only work on leaf nodes + if not node.children: # custom tree model where each node was represented by tuple (name, code,...) => data[1] = code code = self.tree.currentIndex().internalPointer().data[1] arr = eurostat_get(code, cache_dir='__array_cache__') @@ -294,8 +695,6 @@ def add_to_list(self): def view_eurostat_indicator(self, index): - from larray_eurostat import eurostat_get - node = index.internalPointer() title, code, last_update_of_data, last_table_structure_change, data_start, data_end = node.data if not node.children: @@ -313,8 +712,8 @@ def view_eurostat_indicator(self, index): selected_frequencies = dialog.get_selected_frequencies() else: selected_frequencies = current_dataset_labels - - arr = freq_eurostat(selected_frequencies, arr) + + arr = freq_eurostat_editor(selected_frequencies, arr) # Load the data into the editor editor = self.parent() @@ -334,28 +733,13 @@ def keyPressEvent(self, event): def handle_search(self, text): - - def search_term_filter_df(df, text): - terms = text.split() - mask = None - - # First time loop generates a sequence of booleans (lenght = nrows of df) - # Second time it takes 'boolean intersection' with previous search result(s) - for term in terms: - term_mask = (df['Code'].str.contains(term, case=False, na=False)) | (df['Title'].str.contains(term, case=False, na=False)) - if mask is None: - mask = term_mask - else: - mask &= term_mask - return df[mask] - if text: # when text is entered, then hide the treeview and show (new) search results self.tree.hide() self.search_results_list.clear() # filter dataframe based on search term(s) - filtered_df = search_term_filter_df(self.df, text) + filtered_df = multi_search_df(self.df, text) # drop duplicates in search result (i.e. same code, originating from different locations tree) filtered_df = filtered_df.drop_duplicates(subset='Code') @@ -582,7 +966,6 @@ def extract_mat(mat, keys): editor = self.parent().parent() # need to be *two* class-levels higher new_data = editor.data.copy() # make a copy of dataset-dictionary before starting to add new datasets - from larray_eurostat import eurostat_get for code in indicators_as_list: # Adding datasets arr = eurostat_get(code, cache_dir='__array_cache__') # pulls dataset @@ -1087,6 +1470,8 @@ def _setup_file_menu(self, menu_bar): file_menu.addSeparator() file_menu.addAction(create_action(self, _('&Load Example Dataset'), triggered=self.load_example)) file_menu.addAction(create_action(self, _('&Browse Eurostat Datasets'), triggered=self.browse_eurostat)) + file_menu.addAction(create_action(self, _('&Browse AMECO Datasets'), triggered=self.browse_ameco)) + # ============= # # SCRIPTS # # ============= # @@ -1697,6 +2082,13 @@ def browse_eurostat(self): dialog = EurostatBrowserDialog(df, parent=self) dialog.show() + def browse_ameco(self): + with open('larray_editor/ameco_toc_tree.txt', 'r', encoding='utf-8') as file: + ameco_table_of_contents = file.read() + + dialog = AmecoBrowserDialog(ameco_table_of_contents, parent=self) + dialog.show() + class ArrayEditor(AbstractEditor): """Array Editor Dialog""" diff --git a/larray_editor/treemodel.py b/larray_editor/treemodel.py index bd2831c..2b4a7f3 100644 --- a/larray_editor/treemodel.py +++ b/larray_editor/treemodel.py @@ -69,3 +69,42 @@ def rowCount(self, index): node = index.internalPointer() if index.isValid() else self.root return len(node.children) + + +## Extra functions for AMECO tree +class Ameco_SimpleTreeNode: + def __init__(self, data, parent=None): + self.data = data + self.parent = parent + self.children = [] + if parent: + parent.children.append(self) + +def Ameco_parse_tree_structure(lines): + root = Ameco_SimpleTreeNode("Root") + prev_node = root + prev_level = -1 + + for line in lines: + level = line.count('>') + name = line.strip('>').strip() + node = Ameco_SimpleTreeNode(name) + + if level > prev_level: + node.parent = prev_node + prev_node.children.append(node) + elif level == prev_level: + node.parent = prev_node.parent + prev_node.parent.children.append(node) + else: + diff = prev_level - level + higher_node = prev_node.parent + for _ in range(diff): + higher_node = higher_node.parent + node.parent = higher_node + higher_node.children.append(node) + + prev_node = node + prev_level = level + + return root \ No newline at end of file