diff --git a/larray_ameco/get_ameco.py b/larray_ameco/get_ameco.py new file mode 100644 index 0000000..4a5355a --- /dev/null +++ b/larray_ameco/get_ameco.py @@ -0,0 +1,116 @@ +import pandas as pd +import larray as la +import zipfile +import os + +# aameco_get +ameco_input_path = 'http://ec.europa.eu/economy_finance/db_indicators/ameco/documents/' +ameco_output_path = '' +ameco_output_filename = 'ameco.csv' + + +def change_label(s): + ls = s.split('.') + nls = [] + nls.append(ls[0]) + nls.append('-'.join(ls[1:5])) + nls.append(ls[5]) + + return '.'.join(nls) + + +def read_ameco(path, ameco_file): + df = pd.read_csv(path + ameco_file, sep=';', na_values=la.nan, skipinitialspace=True) + df.drop(df.columns[len(df.columns) - 1], axis=1, inplace=True) + # change last column to numeric + df[df.columns[-1]] = df[df.columns[-1]].apply(pd.to_numeric, errors='coerce') + + df = df.drop(['COUNTRY', 'SUB-CHAPTER', 'TITLE', 'UNIT'], axis=1) + idx_columns = [x for x in df.columns if not x.isdigit()] + df_var = df.set_index(idx_columns) + la_data = la.from_frame(df_var).rename({1: 'time'}) + la_data = la_data.set_labels('CODE', change_label) + + return la_data + + +def ameco_get(indicator, path='', drop_csv=True): + ''' + + Parameters + ---------- + indicator + path + drop_csv + + Returns + ------- + + ''' + + import urllib.request + + ameco_zip = indicator + '.zip' + url = ameco_input_path + ameco_zip + + file_name, headers = urllib.request.urlretrieve(url, path + ameco_zip) + print(file_name, 'Date', headers['Date'], 'Content-Length:', headers['Content-Length']) + ziplist = unzip_ameco(path, ameco_zip) + os.remove(path + ameco_zip) + for idx, ameco_file in enumerate(ziplist): + if idx == 0: + la_data = read_ameco(path, ameco_file) + else: + la_data2 = read_ameco(path, ameco_file) + la_data = la_data.append('CODE', la_data2) + + if drop_csv: + os.remove(path + ameco_file) + + la_data = la_data.rename({'CODE': 'RCT'}) + return la_data + + +def unzip_ameco(path, ameco_zip): + print('Unzipping', path + ameco_zip) + namelist = [] + with zipfile.ZipFile(path + ameco_zip, 'r') as zip_ref: + zip_ref.extractall(path) + namelist = zip_ref.namelist() + + return namelist + + +def reshape_and_dump_ameco_dataset(): + import larray as la + ameco_data = ameco_get('ameco0', path='__array_cache__/') + ameco_data = ameco_data.split_axes(sep='.') + + # >>> The RCT Axis contains data of the form 'DEU.1-0-0-0.NPTD'. + # >>> Objective is to split this into three distinct axes: Var, Country, and Numo. + + # Firs handle special case where '_' interferes with our splitting logic. + # Replace 'D_W' with a placeholder 'D°°°W' + ameco_data = ameco_data.set_labels('RCT', lambda label: label.replace('D_W', 'D°°°W')) + + # ROUND 1 + # Split and reformat the string to prepare for the first split. + # The format becomes "B_C|A" in preparation for a split on '_'. + ameco_data = ameco_data.set_labels('RCT', lambda label: (lambda parts: f"{parts[1]}_{parts[2]}|{parts[0]}")(label.split('.'))) + axis_combined_NVC = ameco_data.axes[0].rename('numo_VarCountry') + axis_time = ameco_data.axes[1] + ameco_partially_cleaned = la.Array(ameco_data, axes=[axis_combined_NVC, axis_time]).split_axes(axis_combined_NVC) + + # ROUND 2 + # Modify labels from "C|A" to "C_A" to prepare for the second split on '_'. + ameco_partially_cleaned = ameco_partially_cleaned.set_labels('VarCountry', lambda x: x.replace('|', '_')) + axis_numo = ameco_partially_cleaned.axes[0] + axis_combined_VC = ameco_partially_cleaned.axes[1].rename('Var_Country') + axis_time = ameco_partially_cleaned.axes[2] + ameco_final_cleaned = la.Array(ameco_partially_cleaned, axes=[axis_numo, axis_combined_VC, axis_time]).split_axes(axis_combined_VC) + + # Restore the special case 'D°°°W' back to 'D_W'. + ameco_final_cleaned = ameco_final_cleaned.set_labels('Country', {'D°°°W': 'D_W'}) + + ameco_final_cleaned.to_hdf('__array_cache__/data.h5', 'ameco0') + print('file dumped to __array_cache__/data.h5') diff --git a/larray_editor/ameco_toc_tree.txt b/larray_editor/ameco_toc_tree.txt new file mode 100644 index 0000000..e984025 --- /dev/null +++ b/larray_editor/ameco_toc_tree.txt @@ -0,0 +1,539 @@ +> Population and employment +>> Population +>>> Total population (NPTN) +>>> Total population (National accounts) (NPTD) +>>> Population: 0 to 14 years (NPCN) +>>> Population: 15 to 64 years (NPAN) +>>> Population: 65 years and over (NPON) +>> Labour force +>>> Total labour force (Labour force statistics) (NLTN) +>>> Civilian labour force (Labour force statistics) (NLCN) +>>> Civilian employment, persons (domestic) (NECD) +>>> Civilian employment, persons (national) (NECN) +>> Employment total economy, persons +>>> Employment, persons-; total economy (National accounts) (NETN) +>>> Employment, persons-;all domestic industries (National accounts) (NETD) +>>> Number of self-employed-; total economy (National accounts) (NSTD) +>>> Employees, persons-; total economy (National accounts) (NWTN) +>>> Employees, persons-; all domestic industries (National accounts) (NWTD) +>> Employment total economy, full-time equivalents +>>> Employees, full-time equivalents-; total economy (National accounts) (FWTD) +>>> Employment, full-time equivalents-; total economy (National accounts) (FETD) +>> Unemployment +>>> Total unemployment; Member States: definition -Eurostat (NUTN) +>>> Unemployment rate, total (percentage of civilian labour force); Member States: definition Eurostat (ZUTN) +> Consumption +>> Actual individual final consumption of households +>>> Actual individual final consumption of households at current prices (UCTH) +>>> Actual individual final consumption of households at constant prices (OCTH) +>>> Price deflator actual individual final consumption of households (PCTH) +>> Private final consumption expenditure +>>> Private final consumption expenditure at current prices (UCPH) +>>> Private final consumption expenditure at constant prices (OCPH) +>>> Price deflator private final consumption expenditure (PCPH) +>> Total final consumption expenditure of general government +>>> Final consumption expenditure of general government at current prices (UCTG) +>>> Final consumption expenditure of general government at constant prices (OCTG) +>>> Price deflator total final consumption expenditure of general government (PCTG) +>> Collective consumption expenditure of general government +>>> Collective consumption of general government at current prices (UCCG) +>>> Collective consumption of general government at constant prices (OCCG) +>>> Price deflator collective consumption of general government (PCCG) +>> Individual consumption expenditure of general government +>>> Individual consumption of general government at current prices (UCIG) +>>> Individual consumption of general government at constant prices (OCIG) +>>> Price deflator individual consumption of general government (PCIG) +>> Total consumption +>>> Total consumption expenditure at current prices (UCNT) +>>> Total consumption expenditure at constant prices (OCNT) +>>> Price deflator total consumption (PCNT) +>> Consumer price index +>>> Harmonised consumer price index (1996 = 100) (ZCPIH) +>>> National consumer price index (ZCPIN) +> Capital formation and saving, total economy and sectors +>> Gross fixed capital formation +>>> Gross fixed capital formation at constant prices; total economy (OIGT) +>>> Price deflator gross fixed capital formation; total economy (PIGT) +>>> Gross fixed capital formation at current prices; general government (UIGG) +>>> Gross fixed capital formation at current prices; private sector (UIGP) +>>> Gross fixed capital formation at current prices; total economy (UIGT) +>> Net fixed capital formation +>>> Net fixed capital formation at constant prices; total economy (OINT) +>>> Net fixed capital formation at current prices; general government (UING) +>>> Net fixed capital formation at current prices; private sector (UINP) +>>> Net fixed capital formation at current prices; total economy (UINT) +>> Consumption of fixed capital +>>> Consumption of fixed capital- at constant prices; total economy; Price deflator gross fixed capital formation (OKCT) +>>> Consumption of fixed capital at current prices; general government (UKCG) +>>> Consumption of fixed capital at current prices; total economy (UKCT) +>> Change in inventories +>>> Changes in inventories and acquisitions less disposals of valuables at constant prices; total economy (OIST) +>>> Changes in inventories and acquisitions less disposals of valuables at current prices; total economy (UIST) +>> Gross capital formation +>>> Gross capital formation at constant prices; total economy (OITT) +>>> Gross capital formation at current prices; total economy (UITT) +>> Gross saving +>>> Gross national saving (USGN) +>>> Gross saving: private sector; EU-Member States: ESA 1995 (USGP) +>>> Gross saving-; private sector; EU-Member States: former definition (USGPF) +>> Net saving +>>> Net saving; general government ; EU-Member States: ESA 1995 (USNG) +>>> Net saving: general government; EU-Member States: former definition (USNGF) +>>> Net national saving (USNN) +>>> Net saving-; private sector; EU-Member States: ESA 1995 (USNP) +>>> Net saving; private sector; EU-Member States: former definition (USNPF) +> Gross fixed capital formation by type of goods +>> Construction +>>> Gross fixed capital formation at constant prices; construction (OIGCO) +>>> Price deflator gross fixed capital formation; construction (PIGCO) +>>> Gross fixed capital formation at current prices; construction (UIGCO) +>> Dwellings +>>> Gross fixed capital formation at constant prices; dwellings (OIGDW) +>>> Price deflator gross fixed capital formation; dwellings (PIGDW) +>>> Gross fixed capital formation at current prices; dwellings (UIGDW) +>> Non-residential construction and civil engineering +>>> Gross fixed capital formation at constant prices-; non-residential construction and civil engineering (OIGNR) +>>> Price deflator gross fixed capital formation-; non-residential construction and civil engineering (PIGNR) +>>> Gross fixed capital formation at current prices-; non-residential construction and civil engineering (UIGNR) +>> Equipment +>>> Gross fixed capital formation at constant prices-; equipment (OIGEQ) +>>> Price deflator gross fixed capital formation-; equipment (PIGEQ) +>>> Gross fixed capital formation at current prices; equipment (UIGEQ) +>> Metal products and machinery +>>> Gross fixed capital formation at constant prices; metal products and machinery (OIGMA) +>>> Price deflator gross fixed capital formation; metal products and machinery (PIGMA) +>>> Gross fixed capital formation at current prices; metal products and machinery (UIGMA) +>> Transport equipment +>>> Gross fixed capital formation at constant prices transport equipment (OIGTR) +>>> Price deflator gross fixed capital formation; transport equipment (PIGTR) +>>> Gross fixed capital formation at current prices; transport equipment (UIGTR) +> Domestic and final demand +>> Domestic demand excluding stocks +>>> Domestic demand excluding stocks at constant prices (OUNF) +>>> Price deflator domestic demand excluding stocks (PUNF) +>>> Domestic demand excluding stocks at current prices (UUNF) +>> Domestic demand including stocks +>>> Domestic demand including stocks at constant prices (OUNT) +>>> Price deflator domestic demand including stocks (PUNT) +>>> Domestic demand including stocks at current prices (UUNT) +>>> Domestic demand including stocks at constant prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OUNTQ) +>>> Domestic demand including stocks at constant prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OUNTQ) +>> Final demand +>>> Final demand at constant prices (OUTT) +>>> Price deflator final demand (PUTT) +>>> Final demand at current prices (UUTT) +>> Contributions to the change of the deflator of final demand +>>> Contribution to the change of the final demand deflator of import prices (YPUT0) +>>> Contribution to the change of the final demand deflator of import prices excluding nominal effective exchange rates (YPUT1) +>>> Contribution to the change of the final demand deflator of nominal effective exchange rates (YPUT2) +>>> Contribution to the change of the final demand deflator of relative unit labour costs in national currency (YPUT3) +>>> Contribution to the change of the final demand deflator of domestic factors (YPUT5) +>>> Contribution to the change of the final demand deflator of the GDP price deflator (YPUT6) +>>> Contribution to the change of the final demand deflator of nominal unit labour costs (YPUT7) +>>> Contribution to the change of the final demand deflator of net indirect taxes (YPUT8) +>>> Contribution to the change of the final demand deflator of gross operating surplus; excluding imputed compensation of self-employed (YPUT9) +> National product and income +>> Gross national income +>>> Gross national -income at constant market prices, deflator GDP (OVGN) +>>> Gross national -income at current market prices (UVGN) +>> Gross national disposable income +>>> Gross national disposable income (UVGT) +>> Net national income +>>> National income at current market prices (UVNN) +>> Net national disposable income +>>> National disposable income [net] (UVNT) +>> Domestic product and income, total economy +>>> Gross domestic product (GDP) +>>> Gross domestic product at constant market prices (OVGD) +>>> Price deflator gross domestic product at market prices (PVGD) +>>> Gross domestic product at current market prices (UVGD) +>>> Gross domestic product at current factor cost (UYGD) +>>> Gross domestic product at constant market prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OVGDQ) +>>> Price deflator gross domestic product at market prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PVGDQ) +>>> Gross domestic product at constant market prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OVGDQ) +>>> Price deflator gross domestic product at market prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PVGDQ) +>>> GDP (reference level for excessive deficit procedure) +>>> Gross domestic product at current market prices ; Reference level for excessive deficit procedure (UVGDH) +>> GDP adjusted for the impact of terms of trade +>>> Gross domestic product at constant market prices adjusted for the impact of terms of trade (OVGDA) +>> Trend GDP at constant market prices +>>> Trend gross domestic product at constant market prices (OVGDT) +>>> Gap between actual and trend gross domestic product at constant market prices (AVGDGT) +>> Potential GDP at constant market prices +>>> Potential gross domestic product at constant market prices (OVGDP) +>>> Gap between actual and potential gross domestic product at constant market prices (AVGDGP) +>> Contributions to the change of GDP +>>> Contribution to the increase of GDP at constant market prices of private consumption (CVGD0) +>>> Contribution to the increase of GDP at constant market prices of public consumption (CVGD1) +>>> Contribution to the increase of GDP at constant market prices of total consumption (CVGD10) +>>> Contribution to the increase of GDP at constant market prices of gross fixed capital formation (CVGD2) +>>> Contribution to the increase of GDP at constant market prices of domestic demand excluding stocks (CVGD3) +>>> Contribution to the increase of GDP at constant market prices of net stockbuilding (CVGD4) +>>> Contribution to the increase of GDP at constant market prices of domestic demand including stocks (CVGD5) +>>> Contribution to the increase of GDP at constant market prices of exports of goods and services ; excluding intra-EU trade (CVGD6A) +>>> Contribution to the increase of GDP at constant market prices of final demand ; excluding intra-EU trade (CVGD7A) +>>> Contribution to the increase of GDP at constant market prices of imports of goods and services ; excluding intra-EU trade (CVGD8A) +>>> Contribution to the increase of GDP at constant market prices of the balance of goods and services (CVGD9) +>> Net domestic product +>>> Domestic income at current market prices (UVND) +>>> Domestic income at current factor cost (UYND) +>> Gross value added excluding FISIM +>>> Gross value added at constant basic prices excluding FISIM-; total economy (OVGE) +>>> Gross value added at current basic prices excluding FISIM-; total economy (UVGE) +>> Taxes linked to imports and production and subsidies +>>> Taxes linked to imports and production minus subsidies-; total economy (UTVN) +>>> Taxes linked to imports and production-; total economy (UTVT) +>>> Subsidies-; total economy (UYVT) +>> Gross operating surplus +>>> Gross operating surplus-; total economy (UOGD) +>>> Gross operating surplus-; total economy; adjusted for imputed compensation of self-employed (UQGD) +>> Net operating surplus +>>> Net operating surplus-; total economy (UOND) +>>> Net operating surplus-; total economy-; adjusted for imputed compensation of self-employed (UQND) +> Gross domestic product per head +>> GDP at current market prices per head of population +>>> Gross domestic product at current market prices per head of population (HVGDP) +>>> Gross domestic product at current market prices per head of population (HVGDPR) +>> GDP at current market prices per person employed +>>> Gross domestic product at current market prices per person employed (HVGDE) +>> GDP at constant market prices per head of population +>>> Gross domestic product at constant market prices per head of population (RVGDP) +>> GDP at constant market prices per person employed +>>> Gross domestic product at constant market prices per person employed (RVGDE) +>>> Gross domestic product at constant market prices per person employed ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (RVGDEQ) +>>> Gross domestic product at constant market prices per person employed ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (RVGDEQ) +>> GDP at constant market prices (adjusted for the impact of terms of trade) per head of population +>>> GDP at constant market prices adjusted for the impact of terms of trade per head of population (RVGDAP) +>> GDP at constant market prices (adjusted for the impact of terms of trade) per person employed +>>> GDP at constant market prices adjusted for the impact of terms of trade per person employed (RVGDAE) +> Labour costs; total economy +>> Compensation of employees +>>> Compensation of employees; total economy (UWCD) +>> Nominal compensation per employee +>>> Nominal compensation per employee: total economy (HWCDW) +>>> Nominal compensation per employee: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (HWCDWQ) +>>> Nominal compensation per employee: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (HWCDWQ) +>> Real compensation per employee +>>> Real compensation per employee, deflator private consumption; total economy (RWCDC) +>>> Real compensation per employee, deflator GDP; total economy (RWCDV) +>>> Real compensation per employee, deflator GDP: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (RWCDVQ) +>>> Real compensation per employee, deflator GDP: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (RWCDVQ) +>> Adjusted wage share +>>> Adjusted wage share total economy (ALCD0) +>>> Adjusted wage share; total economy (ALCD2) +>> Nominal unit labour costs +>>> Nominal unit labour costs; total economy (PLCD) +>>> Nominal unit labour costs: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PLCDQ) +>>> Nominal unit labour costs: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PLCDQ) +>> Real unit labour costs +>>> Real unit labour costs total economy (QLCD) +>>> Real unit labour costs: total economy ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (QLCDQ) +>>> Real unit labour costs: total economy ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (QLCDQ) +> Capital stock +>> Net capital stock at constant prices; total economy +>>> Net capital stock at constant prices; total economy (OKND) +>> Net capital stock at constant prices per person employed +>>> Net capital stock at constant prices per person employed; total economy (Capital intensity) (RKNDE) +>> GDP at constant market prices per unit of net capital stock; total economy +>>> Gross domestic product at constant market prices per unit of net capital stock (Capital productivity) (AVGDK ) +>>> Factor productivity (Total, labour share and capital share) +>>> Labour share in total factor productivity: total economy (ZVGDE) +>>> Total factor productivity: total economy (ZVGDF) +>>> Capital share in total factor productivity: total economy (ZVGDK) +>> Labour-capital substitution +>>> Labour-capital substitution: total economy (ZKNDE) +>> Capital-labour substitution +>>> Capital-labour substitution: total economy (ZEKND) +>> Marginal efficiency of capital +>>> Marginal efficiency of capital; total economy (AKGDV) +>> Net returns on net capital stock +>>> Net returns on net capital stock; total economy (APNDK) +> Exports and imports of goods and services +>> Exports of goods and services +>>> Exports of goods and services at constant prices (OXGS) +>>> Price deflator exports of goods and services (PXGS) +>>> Exports of goods and services at current prices (National accounts) (UXGS) +>>> Export markets: Export weighted imports ; Goods and services at constant prices: 23 industrial markets (EU-14, CHE NOR USA CAN JAP AUS MEX NZL and TUR) (VMGSW) +>>> Market performance of exports of goods and services on export weighted imports of goods and services ; 23 industrial markets (EU-14, CHE NOR USA CAN JAP AUS MEX NZL and TUR) (VXGSP) +>>> Exports of goods and services at constant prices ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (OXGSQ) +>>> Price deflator exports of goods and services ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (PXGSQ) +>>> Exports of goods and services at constant prices ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (OXGSQ) +>>> Price deflator exports of goods and services ; Performance relative to the rest of 22 industrial countries: double export weights : EU-14, CHE NOR USA CAN JAP AUS MEX and NZL (PXGSQ) +>> Imports of goods and services +>>> Imports of goods and services at constant prices (OMGS) +>>> Price deflator imports of goods and services (PMGS) +>>> Imports of goods and services at current prices (National accounts) (UMGS) +>>> Terms of trade (goods and services) +>>> Terms of trade goods and services (National accounts; 1995 = 100) (APGS) +>>> Impact of terms of trade goods and services on real income (APTA) +>> Exports of goods +>>> Exports of goods at constant prices (OXGN) +>>> Price deflator exports of goods (PXGN) +>>> Exports of goods at current prices (National accounts) (UXGN) +>> Imports of goods +>>> Imports of goods at constant prices (OMGN) +>>> Price deflator imports of goods (PMGN) +>>> Imports of goods at current prices (National accounts) (UMGN) +>>> Terms of trade (goods) +>>> Terms of trade goods (National accounts) (APGN) +> Balances with the rest of the world +>> Net exports of goods and services +>>> Net exports of goods and services at current prices (National accounts) (UBGS) +>> Net exports of goods +>>> Net exports of goods at current prices (National accounts) (UBGN) +>> Balance of primary income from the rest of the world +>>> Net factor income from the Rest of the world (National accounts) (UBRA) +>> Balance of current transfers with the rest of the world +>>> Net current transfers from the rest of the world (National accounts) (UBTA) +>> Balance on current transactions with the rest of the world +>>> Balance on current transactions with the rest of the world (National accounts) (UBCA) +>> Balance of capital transfers with the rest of the world +>>> Net capital transactions with the rest of the world (National accounts) (UBKA) +>> Net lending (+) or net borrowing (-) of the nation +>>> Net lending (+) or net borrowing (-); total economy (UBLA) +> Foreign trade at current prices +>> Exports goods (FOB) total +>>> Total exports of goods (FOB); Foreign trade statistics (DXGT) +>>> Total exports of goods excluding energy products; Foreign trade statistics (DXGT3) +>> Exports goods (FOB) intra EU +>>> Intra EU exports of goods; Foreign trade statistics (DXGI) +>>> Intra EU exports of goods excluding energy products; Foreign trade statistics (DXGI3) +>> Exports goods (FOB) extra EU +>>> Extra EU exports of goods; Foreign trade statistics (DXGE) +>>> Extra EU exports of goods excluding energy products; Foreign trade statistics (DXGE3) +>> Imports goods (CIF) total +>>> Total imports of goods (CIF) ; Foreign trade statistics (DMGT) +>>> Total imports of goods excluding energy products; Foreign trade statistics (DMGT3) +>> Imports goods (CIF) intra EU +>>> Intra EU imports of goods (CIF); Foreign trade statistics (DMGI) +>>> Intra EU imports of goods excluding energy products; Foreign trade statistics (DMGI3) +>> Imports goods (CIF) extra EU +>>> Extra EU imports of goods; Foreign trade statistics (DMGE) +>>> Extra EU imports of goods excluding energy products; Foreign trade statistics (DMGE3) +>> Foreign trade shares in world trade +>>> Average share of imports and exports of goods in world trade excluding intra EU trade; Foreign trade statistics (AAGE) +>>> Average share of imports and exports of goods in world trade including intra EU trade; Foreign trade statistics (AAGT) +>>> Share of imports of goods in world imports excluding intra EU imports ; Foreign trade statistics (AMGE) +>>> Share of imports of goods in world imports including intra EU imports-; Foreign trade statistics (AMGT) +>>> Share of exports of goods in world exports excluding intra EU exports; Foreign trade statistics (AXGE) +>>> Share of exports of goods in world exports including intra EU exports; Foreign trade statistics (AXGT) +>>> National accounts by branch of activity (Part I) +>> Occupied population +>>> Employment, full-time equivalents agriculture, forestry and fishery products (National accounts) (FET1) +>>> Employment, full-time equivalents; industry excluding building and construction (National accounts) (FET2) +>>> Employment, full-time equivalents; building and construction (National accounts) (FET4) +>>> Employment, full-time equivalents; services (National accounts) (FET5) +>>> Employment, full-time equivalents; manufacturing industry (National accounts) (FETM) +>>> Employment, persons; agriculture, forestry and fishery products (National accounts) (NET1) +>>> Employment, persons; industry excluding building and construction (National accounts) (NET2) +>>> Employment, persons; building and construction (National accounts) (NET4) +>>> Employment, persons; services (National accounts) (NET5) +>>> Employment, persons; manufacturing industry (National accounts) (NETM) +>> Wage and salary earners +>>> Employees, full-time equivalents; agriculture, forestry and fishery products (National accounts) (FWT1) +>>> Employees, full-time equivalents; industry excluding building and construction (National accounts) (FWT2) +>>> Employees, full-time equivalents; building and construction (National accounts) (FWT4) +>>> Employees, full-time equivalents; services (National accounts) (FWT5) +>>> Employees, full-time equivalents; manufacturing industry (National accounts) (FWTM) +>>> Employees, persons: agriculture, forestry and fishery products (National accounts) (NWT1) +>>> Employees, persons; industry excluding building and construction (National accounts) (NWT2) +>>> Employees, persons; building and construction (National accounts) (NWT4) +>>> Employees, persons; services (National accounts) (NWT5) +>>> Employees, persons; manufacturing industry (National accounts) (NWTM) +>> Gross value added at current prices +>>> Gross value added at current prices; total of branches (UVG0) +>>> Gross value added at current prices; agriculture, forestry and fishery products (UVG1) +>>> Gross value added at current prices; industry excluding building and construction (UVG2) +>>> Gross value added at current prices; building and construction (UVG4) +>>> Gross value added at current prices; services (UVG5) +>>> Gross value added at current prices; manufacturing industry (UVGM) +>> Gross value added at current prices per person employed +>>> Gross value added at current prices per person employed; agriculture, forestry and fishery products (HVG1E) +>>> Gross value added at current prices per person employed; industry excluding building and construction (HVG2E) +>>> Gross value added at current prices per person employed; building and construction (HVG4E) +>>> Gross value added at current prices per person employed; services (HVG5E) +>>> Gross value added at current prices per person employed; manufacturing industry (HVGME) +>> Gross value added at current prices per employee +>>> Gross value added at current prices per employee; manufacturing industry (HVGMW) +>> Gross value added at constant prices +>>> Gross value added at constant prices; total of branches (OVG0) +>>> Gross value added at constant prices; agriculture, forestry and fishery products (OVG1) +>>> Gross value added at constant prices industry excluding building and construction (OVG2) +>>> Gross value added at constant prices; building and construction (OVG4) +>>> Gross value added at constant prices; services (OVG5) +>>> Gross value added at constant prices; manufacturing industry (OVGM) +>>> National accounts by branch of activity (Part II) +>> Gross value added at constant prices per person employed +>>> Real unit labour costs; manufacturing industry (QLCM) +>>> Gross value added at constant prices per person employed; agriculture, forestry and fishery products (RVG1E) +>>> Gross value added at constant prices per person employed; industry excluding building and construction (RVG2E) +>>> Gross value added at constant prices per person employed; building and construction (RVG4E) +>>> Gross value added at constant prices per person employed; services (RVG5E) +>>> Gross value added at constant prices per person employed; manufacturing industry (RVGME) +>> Gross value added at constant prices per employee +>>> Gross value added at constant prices per employee; manufacturing industry (RVGMW) +>> Price deflator gross value added +>>> Price deflator gross value added; agriculture, forestry and fishery products (PVG1) +>>> Price deflator gross value added; industry excluding building and construction (PVG2) +>>> Price deflator gross value added; building and construction (PVG4) +>>> Price deflator gross value added; services (PVG5) +>>> Price deflator gross value added; manufacturing industry (PVGM) +>> Industrial production +>>> Industrial production, construction excluded (VPRI) +>> Compensation of employees +>>> Compensation of employees; agriculture, forestry and fishery products (UWC1) +>>> Compensation of employees; industry excluding building and construction (UWC2) +>>> Compensation of employees; building and construction (UWC4) +>>> Compensation of employees; services (UWC5) +>>> Compensation of employees; manufacturing industry (UWCM) +>> Nominal compensation per employee +>>> Nominal compensation per employee; agriculture, forestry and fishery products (HWC1W) +>>> Nominal compensation per employee; industry excluding building and construction (HWC2W) +>>> Nominal compensation per employee; building and construction (HWC4W) +>>> Nominal compensation per employee; services (HWC5W) +>>> Nominal compensation per employee; manufacturing industry (HWCMW) +>>> Adjusted wage share (manufacturing) +>>> Adjusted wage share; manufacturing industry (ALCM) +>> Unit labour and wage costs +>>> Nominal unit wage costs; agriculture, forestry and fishery products (1995 = 100) (PWC1) +>>> Nominal unit wage costs; industry excluding building and construction (1995 = 100) (PWC2) +>>> Nominal unit wage costs; building and construction (1995 = 100) (PWC4) +>>> Nominal unit wage costs; services (1995 = 100) (PWC5) +>>> Nominal unit wage costs; manufacturing industry (1995 = 100) (PWCM) +>>> Nominal unit labour costs; manufacturing industry (1995) (PLCM) +> Monetary variables +>> Nominal exchange rates +>>> ECU-EUR exchange rates (XNE) +>> Nominal exchange rates +>>> Conversion rates between euro and former euro-zone national currencies (XNEF) +>> Nominal effective exchange rates +>>> Nominal effective exchange rates; Performance relative to the rest of 35 industrial countries; double export weights (XUNNQ) +>> Effective exchange rates +>>> Real effective exchange rates ; Performance relative to the rest of 14 EU countries (EU-15 excl. L): double export weights (XUNRQ) +>> GDP purchasing power parities +>>> GDP purchasing power parities (KNP) +>> Interest rates +>>> Nominal long-term interest rates (ILN) +>>> Real long-term interest rates, deflator private consumption (ILRC) +>>> Real long-term interest rates, deflator GDP (ILRV) +>>> Nominal short-term interest rates (ISN) +>>> Real short-term interest rates, deflator private consumption (ISRC) +>>> Real short-term interest rates, deflator GDP (ISRV) +>>> Yield curve (IYN) +> Non-financial and financial corporations (S11 + S12) +>> Revenue +>>> Other subsidies on production; corporations (UYVC) +>>> Net current transfers received; corporations (UCTRC) +>>> Net property income; corporations (UYNC) +>> Expenditure +>>> Adjustment for the change in net equity of households in pension funds; corporations (UEHC) +>>> Gross capital formation; corporations (UITC) +>>> Other capital expenditure, net; corporations (UKOC) +>>> Other taxes on production; corporations (UTVC) +>>> Current taxes on income and wealth; corporations (UTYC) +>>> Compensation of employees-; corporations (UWCC) +>> Balances +>>> Gross value added at basic prices; corporations (UGVAC) +>>> Net lending (+) or net borrowing (-); corporations (UBLC) +>>> Gross saving; corporations (USGC) +>>> Net saving; corporations (USNC) +>>> Gross disposable income; corporations (UVGC) +>>> Net disposable income; corporations (UVNC) +>>> Gross operating surplus; corporations (UOGC) +>>> Gross balance of primary income; corporations (UBGC) +>>> Net balance of primary income; corporations (UBNC) +> Households and NPISH (S14 + S15) +>> Revenue +>>> Compensation of employees; households and NPISH (UWCH) +>>> Gross wages and salaries; households and NPISH (UWSH) +>>> Non-labour income; households and NPISH (UYOH) +>>> Net property income; households and NPISH (UYNH) +>>> Current transfers received; households and NPISH (UCTRH) +>> Expenditure +>>> Net lending (+) or net borrowing (-); households and NPISH (UBLH) +>>> Other capital expenditure, net; households and NPISH (UKOH) +>>> Gross saving; households and NPISH (USGH) +>>> Current taxes on income and wealth; households and NPISH (UTYH) +>>> Current transfers paid; households and NPISH (UCTPH) +>> Balances +>>> Saving rate, gross; households and NPISH (ASGH) +>>> Real gross disposable income, deflator private consumption; households and NPISH (OVGH) +>>> Net saving; households (USNH) +>>> Gross operating surplus and mixed income; households and NPISH (UOGH) +>>> Gross disposable income; households and NPISH (UVGH) +>>> Final consumption expenditure; households and NPISH (UCPH0) +>>> Gross capital formation; households and NPISH (UITH) +> General government (S13) +>> Revenue +>>> Capital transfers received; general government; ESA 1995 (UKTTG) +>>> Total revenue; general government; ESA 1995 (URTG) +>>> Total tax burden excluding imputed social security contributions; total economy; ESA 1995 (UTAT) +>>> Total tax burden including imputed social security contributions; total economy; ESA 1995 (UTTT) +>>> Taxes linked to imports and production (indirect taxes); general government; ESA 1995 (UTVG) +>> Taxes on production and imports comprise: +>>> Current taxes on income and wealth (direct taxes); general government; ESA 1995 (UTYG) +>>> Other current revenue; general government; ESA 1995 (UROG) +>>> Actual social contributions received; general government; ESA 1995 (UTAG) +>>> Imputed social contributions; general government; ESA 1995 (UTIG) +>>> Current tax burden; total economy; ESA 1995 (UTCT) +>>> Capital taxes; general government; ESA 1995 (UTKG) +>>> Social contributions received; general government; ESA 1995 (UTSG) +>>> Total current revenue; general government; ESA 1995 (URCG) +>> Expenditure +>>> Implicit interest rate; general government; -ESA 1995 (AYIGD) +>>> Final consumption expenditure of general government; ESA 1995 (UCTG0) +>>> Interest; general government; ESA 1995 (UYIG) +>>> Interest- including flows on swaps and FRAs (Forward Rate Agreements); general government; Excessive deficit procedure (UYIGE) +>>> Social transfers other than in kind; general government-; ESA 1995 (UYTGH) +>>> Subsidies; general government; ESA 1995 (UYVG) +>>> Real total expenditure of general government, deflator GDP; ESA 1995 (OUTG) +>>> Real total expenditure excluding interest of general government, deflator GDP; ESA 1995 (OUTGI) +>>> Collective consumption expenditure; ESA 1995 (UCCG0) +>>> Total current expenditure excluding interest; general government; ESA 1995 (UUCGI) +>>> Total expenditure excluding interest; general government; ESA 1995 (UUTGI) +>>> Social transfers in kind; ESA 1995 (UCIG0) +>>> Compensation of employees; general government; ESA 1995 (UWCG) +>>> Other current expenditure; general government; ESA 1995 (UUOG) +>>> Total current expenditure; general government; ESA 1995 (UUCG) +>>> Total current expenditure; general government; Excessive deficit procedure (UUCGE) +>>> Capital transfers paid; general government; ESA 1995 (UKTGT) +>>> Gross fixed capital formation; general government; ESA 1995 (UIGG0) +>>> Total expenditure; general government; ESA 1995 (UUTG) +>>> Total expenditure; general government; Excessive deficit procedure (UUTGE) +>>> Other capital expenditure, including capital transfers; general government; ESA 1995 (UKOG) +>> Balances +>>> Net lending (+) or net borrowing (-); general government; ESA 1995 (UBLG) +>>> Net lending (+) or net borrowing (-); general government; Excessive deficit procedure (UBLGE) +>>> Net lending (+) or net borrowing (-) excluding interest; general government; ESA 1995 (UBLGI) +>>> Net lending (+) or net borrowing (-) excluding interest: general government; Excessive deficit procedure (UBLGIE) +>>> Gross saving; general government; ESA 1995 (USGG) +>>> Gross saving; general government; Excessive deficit procedure (USGGE) +>>> Gross disposable income; general government; ESA 1995 (UVGG) +>>> Net disposable income; general government; ESA 1995 (UVNG) +>>> Net lending (+) or net borrowing (-) excluding gross fixed capital formation; general government; ESA 1995 (UBLGG) +>>> Public finance (other variables) +>> Cyclical adjustment of public finance variables based on TREND GDP +>>> Cyclically adjusted net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGA) +>>> Net lending (+) or net borrowing (-) excluding interest of general government adjusted for the cyclical component; ESA 1995 (UBLGB) +>>> Cyclical component of net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGC) +>>> Cyclically adjusted total revenue of general government; ESA 1995 (URTGA) +>>> Cyclical component of revenue of general government; ESA 1995 (UTCGC) +>>> Cyclical component of expenditure of general government (UUCGC) +>>> Cyclically adjusted total expenditure of general government; ESA 1995 (UUTGA) +>>> Total expenditure excluding interest of general government adjusted for the cyclical component; ESA 1995 (UUTGB) +>> Cyclical adjustment of public finance variables based on POTENTIAL GDP +>>> Cyclically adjusted net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGAP) +>>> Net lending (+) or net borrowing (-) excluding interest of general government adjusted for the cyclical component; ESA 1995 (UBLGBP) +>>> Cyclical component of net lending (+) or net borrowing (-) of general government; ESA 1995 (UBLGCP) +>>> Cyclically adjusted total revenue of general government; ESA 1995 (URTGAP) +>>> Cyclical component of revenue of general government; ESA 1995 (UTCGCP) +>>> Cyclical component of expenditure of general government (UUCGCP) +>>> Cyclically adjusted total expenditure of general government; ESA 1995 (UUTGAP) +>>> Total expenditure excluding interest of general government adjusted for the cyclical component; ESA 1995 (UUTGBP) +>> Gross public debt +>>> Snow ball effect on general government consolidated gross debt; ESA 1995 (ADGGI) +>>> Impact of the nominal increase of GDP on general government consolidated gross debt; ESA 1995 (ADGGU) +>>> Stock-flow adjustment on general government consolidated gross debt; ESA 1995 (UDGGS) +>>> General government consolidated gross debt; ESA 1995 (UDGG) +>>> General government consolidated gross debt ; EU-Member States: ESA 95 and former definition (linked series) (UDGGL) +>>> General government consolidated gross debt; ESA 1995 (UDGGR) \ No newline at end of file diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 6ed17a7..abcf11d 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -1,11 +1,15 @@ import io import os import re +import yaml +from datetime import datetime import sys from collections.abc import Sequence from contextlib import redirect_stdout from pathlib import Path from typing import Union +from larray_eurostat import freq_eurostat_editor, eurostat_get +from larray_ameco.get_ameco import reshape_and_dump_ameco_dataset # Python3.8 switched from a Selector to a Proactor based event loop for asyncio but they do not offer the same @@ -26,6 +30,7 @@ import matplotlib import matplotlib.axes import numpy as np +import pandas as pd import larray as la @@ -34,12 +39,16 @@ get_versions, get_documentation_url, urls, RecentlyUsedList) from larray_editor.arraywidget import ArrayEditorWidget from larray_editor.commands import EditSessionArrayCommand, EditCurrentArrayCommand +from larray_editor.treemodel import SimpleTreeNode, SimpleLazyTreeModel, Ameco_parse_tree_structure -from qtpy.QtCore import Qt, QUrl, QSettings + +from qtpy.QtCore import Qt, QUrl, QSettings, QFileInfo, QThread, Signal from qtpy.QtGui import QDesktopServices, QKeySequence from qtpy.QtWidgets import (QMainWindow, QWidget, QListWidget, QListWidgetItem, QSplitter, QFileDialog, QPushButton, - QDialogButtonBox, QShortcut, QVBoxLayout, QGridLayout, QLineEdit, - QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton) + QDialogButtonBox, QShortcut, QVBoxLayout, QHBoxLayout, QGridLayout, QLineEdit, + QCheckBox, QComboBox, QMessageBox, QDialog, QInputDialog, QLabel, QGroupBox, QRadioButton, + QTreeView, QTextEdit, QMenu, QAction, QTreeWidget, QTreeWidgetItem) + try: from qtpy.QtWidgets import QUndoStack @@ -83,6 +92,913 @@ DISPLAY_IN_GRID = (la.Array, np.ndarray) +def multi_search_df(df, text): + terms = text.split() + mask = None + + # Takes 'boolean intersection' with previous search result(s) + for term in terms: + term_mask = (df['Code'].str.contains(term, case=False, na=False)) | (df['Title'].str.contains(term, case=False, na=False)) + if mask is None: + mask = term_mask + else: + mask &= term_mask + return df[mask] + + +def num_leading_spaces(s): + i = 0 + while s[i] == ' ': + i += 1 + return i + + +def indented_df_to_treenode(df, indent=4, indented_col=0, colnames=None, header=None): + if colnames is None: + colnames = df.columns.tolist() + if header is None: + header = [name.capitalize() for name in colnames] + + root = SimpleTreeNode(None, header) + df = df[colnames] + parent_per_level = {0: root} + # iterating on the df rows directly (via df.iterrows()) is too slow + for row in df.values: + row_data = row.tolist() + indented_col_value = row_data[indented_col] + level = num_leading_spaces(indented_col_value) // indent + # remove indentation + row_data[indented_col] = indented_col_value.strip() + + parent_node = parent_per_level[level] + node = SimpleTreeNode(parent_node, row_data) + parent_node.children.append(node) + parent_per_level[level + 1] = node + return root + + + +class FrequencyFilterDialog(QDialog): + def __init__(self, available_labels, parent=None): + super().__init__(parent) + + self.setWindowTitle("Select frequency") + layout = QVBoxLayout(self) + hbox = QHBoxLayout() + + self.checkboxes = {} + label_map = {'M': 'Month', 'Q': 'Quarter', 'A': 'Year', 'W': 'Week', 'D': 'Day'} + + # generate only relevant checkboxes, i.e. based on available_labels argument + for label in available_labels: + checkbox = QCheckBox(label_map[label], self) + checkbox.setChecked(True) + self.checkboxes[label] = checkbox + hbox.addWidget(checkbox) + + layout.addLayout(hbox) + + ok_button = QPushButton("Ok", self) + ok_button.clicked.connect(self.accept) + + layout.addWidget(ok_button) + + # Function to pass back selected frequencies to parent dialog + def get_selected_frequencies(self): + freqs = [] + for label, checkbox in self.checkboxes.items(): + if checkbox.isChecked(): + freqs.append(label) + return freqs + + + +class AmecoDownloadThread(QThread): + # Needs to run in Seperate Thread to avoid blocking the UI + finishedSignal = Signal() + errorSignal = Signal(str) + + def run(self): + print("Inside the download thread...") + try: + reshape_and_dump_ameco_dataset() + print("Finished downloading in the thread.") + self.finishedSignal.emit() + except Exception as e: + errorMsg = f"An unexpected error occurred: {str(e)}" + self.errorSignal.emit(errorMsg) + + + +class AmecoBrowserDialog(QDialog): + def __init__(self, ameco_table_of_contents, parent=None): + super(AmecoBrowserDialog, self).__init__(parent) + + # Basic UI properties + self.title = 'AMECO Database' + self.ameco_table_of_contents = ameco_table_of_contents + + # Initialize UI elements + self.init_ui() + + + def init_ui(self): + self.init_widgets() + self.set_properties() + self.connect_signals_slots() + self.configure_layout() + self.populate_tree(self.ameco_table_of_contents) + self.show() + + + def init_widgets(self): + self.tree = QTreeWidget() + self.search_input = QLineEdit(self) + self.search_results_list = QListWidget() + self.refresh_button = QPushButton("Last Updated: ", self) + self.update_text_for_last_updated_button() # Means here: put initial text for 'Last Update' button + self.downloadThread = AmecoDownloadThread() + + + def set_properties(self): + self.resize(750, 600) + self.setWindowTitle(self.title) + self.tree.setHeaderHidden(True) + self.search_input.setPlaceholderText("Search...") + self.search_results_list.hide() + + + def connect_signals_slots(self): + # Connect for search functionalities + self.search_input.textChanged.connect(self.handle_search) + self.search_results_list.itemClicked.connect(self.handle_search_item_click) + + # Connect for refresh button and finished download + self.refresh_button.clicked.connect(self.start_download) + self.downloadThread.finishedSignal.connect(self.update_text_for_last_updated_button) + + # Connect for treeitem click + self.tree.itemDoubleClicked.connect(self.on_tree_item_clicked) + + + def configure_layout(self): + # Search layout configuration + search_layout = QHBoxLayout() + search_layout.addWidget(self.search_input) + search_layout.addWidget(self.refresh_button) + + # Main layout configuration + layout = QVBoxLayout() + layout.addLayout(search_layout) + layout.addWidget(self.search_results_list) + layout.addWidget(self.tree) + self.setLayout(layout) + + + def populate_tree(self, ameco_table_of_contents): + root_tree_node = Ameco_parse_tree_structure(ameco_table_of_contents.split('\n')) + for child in root_tree_node.children: + self.ameco_add_nodes(self.tree, child) + + + def start_download(self): + if not self.downloadThread.isRunning(): + # Show a popup box to inform the user that the download has started. + msgBox = QMessageBox() + msgBox.setText("Download started. AMECO datasets are published once per year.") + msgBox.setIcon(QMessageBox.Information) + msgBox.setStandardButtons(QMessageBox.Ok) + msgBox.exec_() + + self.downloadThread.start() + self.downloadThread.setPriority(QThread.HighPriority) + + + def update_text_for_last_updated_button(self): + print("Updating 'Last Updated' after download (or init)...") + + # Get the last modified datetime + file_path = '__array_cache__/data.h5' + info = QFileInfo(file_path) + last_modified = info.lastModified().toString("dd-MM-yyyy HH:mm:ss") + + # Update the button's label + self.refresh_button.setText(f"Last Updated: {last_modified}") + + + def ameco_tree_to_dataframe(self, ameco_table_of_contents): + rows = [] + lines = ameco_table_of_contents.split('\n') + for line in lines: + # Match title and code using regex + match = re.match(r'^(.*)\s+\(([^)]+)\)$', line) + if match: + title, code = match.groups() + # Remove prefixes >, >>, and >>> + title = title.replace('>', '').strip() + rows.append({'Title': title, 'Code': code.strip()}) + + df = pd.DataFrame(rows) + return df + + + def ameco_add_nodes(self, qt_parent, tree_node): + qt_node = QTreeWidgetItem(qt_parent, [tree_node.data]) + for child in tree_node.children: + self.ameco_add_nodes(qt_node, child) + + + def handle_search(self, text): + self.df = self.ameco_tree_to_dataframe(self.ameco_table_of_contents) + + if text: + self.tree.hide() + self.search_results_list.clear() + + filtered_df = multi_search_df(self.df, text) + filtered_df = filtered_df.drop_duplicates(subset='Code') + results = [f"{row['Title']} ({row['Code']})" for _, row in filtered_df.iterrows()] + + self.search_results_list.addItems(results) + self.search_results_list.show() + + else: # if search field is empty + self.tree.show() + self.search_results_list.hide() + + + def handle_search_item_click(self, item): + # Assume the substring '(code)' is always the last thing in parentheses + matches = re.findall(r'\(([^)]+)\)', item.text()) + + if matches: + code = matches[-1] + popup = AmecoSettingsDialog(code, self) + popup.exec_() + + + def on_tree_item_clicked(self, item): + # Extracts the code of item clicked and opens next Popup dialog. + if item.childCount() == 0: + import re + match = re.findall(r'\(([^)]+)\)', item.text(0)) # search last substring '(code)' + if match: + var_name = match[-1] + else: + var_name = "" + + # print(f"the extracted code of clicked item is: {var_name}") + popup = AmecoSettingsDialog(var_name, self) + popup.exec_() + + + def keyPressEvent(self, event): + if event.key() in [Qt.Key_Return, Qt.Key_Enter] and self.tree.hasFocus(): + item = self.tree.currentItem() + if item and item.childCount() == 0: + matches = re.findall(r'\(([^)]+)\)', item.text(0)) # search last substring '(code)' + if matches: + code = matches[-1] + popup = AmecoSettingsDialog(code, self) + popup.exec_() + + + +class AmecoSettingsDialog(QDialog): + def __init__(self, var_name, parent=None): + super().__init__(parent) + self.resize(500, 150) + self.setWindowTitle("Settings") + self.load_data(var_name) + self.init_ui(var_name) + + + def load_data(self, var_name): + self.ameco = la.read_hdf("__array_cache__/data.h5", "ameco0") + all_numerical_codes = [str(self.ameco.axes[0][i]) for i in self.ameco.axes[0]] # adding str() is necessary! + + # Given a *particular* var name, most numerical codes deliver NaN arrays. Consider only relevant numerical codes. + relevant_numerical_codes_for_var = [] + for id in all_numerical_codes: + is_all_nan = la.isnan(self.ameco[id, var_name]).all() + + if not is_all_nan: + relevant_numerical_codes_for_var.append([int(id_part) for id_part in id.split("-")]) # "1-2-3-4" [str] -> [1,2,3,4] + + # Put only relevant codes in dropdowns (= i.e. only need subset of the total 27 combo possibilities) + self.choice_list = relevant_numerical_codes_for_var + print(f"All possible choices: {self.choice_list}") # temp use for debugging, in case some codes are not listed in official document + + + def init_ui(self, var_name): + layout = QVBoxLayout() + self.initialize_content_dropdowns() + + self.drop1, self.drop2, self.drop3, self.drop4 = QComboBox(), QComboBox(), QComboBox(), QComboBox() + self.populate_dropdowns() + self.setup_dropdown_grid(layout) + self.setup_action_elements(layout, var_name) + self.setLayout(layout) + + + def initialize_content_dropdowns(self): + # Mapping dictionaries for dropdowns (codes -> description). See AMECO documentation for more info. + self.drop1_options = { + 1: 'Levels (and moving arithmetic mean for time periods)', + 2: 'Levels (and moving geometric mean for time periods)', + 3: 'Index numbers (and moving arithmetic mean for time periods)', + 4: 'Index numbers (and moving geometric mean for time periods)', + 5: 'Annual percentage changes (and moving arithmetic mean for time periods)', + 6: 'Annual percentage changes (and moving geometric mean for time periods)', + 7: 'Absolute value of annual percentage changes (and moving arithmetic mean for time periods)', + 8: 'Moving percentage changes', + 9: 'Annual changes (and moving arithmetic mean for time periods)', + 10: 'Absolute value of annual changes (and moving arithmetic mean for time periods)', + 11: 'Moving changes', + } + + self.drop2_options = { + 0: 'Standard aggregations (data converted to a common currency and summed)', + 1: 'Weighted mean of t/t-1 national ratios, weights current prices in ECU/EUR', + 2: 'Weighted mean of t/t-1 national ratios, weights current prices in PPS', + } + + self.drop3_options = { + 0: 'Original units (e.g. national currency, persons, etc.)', + 99: 'ECU/EUR', + 212: 'PPS (purchasing power standards)', + 300: 'Final demand', + 310: 'Gross domestic product at market prices', + 311: 'Net domestic product at market prices', + 312: 'Gross domestic product at current factor cost', + 313: 'Net domestic product at current factor cost', + 315: 'Trend gross domestic product at market prices', + 316: 'Potential gross domestic product at market prices', + 318: 'Total gross value added', + 319: 'Gross domestic product at market prices (excessive deficit procedure)', + 320: 'Gross national product at market prices', + 321: 'National income at market prices', + 322: 'Gross national disposable income at market prices', + 323: 'National disposable income at market prices', + 338: 'Gross value added at market prices; manufacturing industry', + 380: 'Current revenue general government', + 390: 'Total expenditure general government', + 391: 'Current expenditure general government', + 410: 'Total population, demographic statistics', + 411: 'Population, 15 to 64 years', + 412: 'Total labour Force', + 413: 'Civilian labour force', + 414: 'Population 15 to 74 years', + 420: 'Total population (national accounts)', + } + + self.drop4_options = { + 0: 'value for the reporting country', # default + 215: 'former EU-15', + 315: 'former EU-15', + 327: 'former EU-27', # added extra: not in official docs? but sometimes used, cf. 'HVGDPR' + 328: 'former EU-28', + 415: 'former EU-15 (bis)', # in documentation double code (315), but then issue with reverse dictionary map, so change text slightly + 424: '24 industrial countries: former EU-15 & CH NR TR US CA MX JP AU NZ', + 437: '37 industrial countries: EU-27 & CH NR TR UK US CA MX JP AU NZ', + } + + + def populate_dropdowns(self): + # Some sets to keep track of adding procedure (i.e. useful to avoid double adds) + added_drop1, added_drop2, added_drop3, added_drop4 = set(), set(), set(), set() + + for item in self.choice_list: + # extract fourfold from iterator 'item' (which is a list of 4 values, e.g. [1,0,0,0]) + drop1_val, drop2_val, drop3_val, drop4_val = item + + if drop1_val in self.drop1_options and drop1_val not in added_drop1: + self.drop1.addItem(self.drop1_options[drop1_val]) + added_drop1.add(drop1_val) + + if drop2_val in self.drop2_options and drop2_val not in added_drop2: + self.drop2.addItem(self.drop2_options[drop2_val]) + added_drop2.add(drop2_val) + + if drop3_val in self.drop3_options and drop3_val not in added_drop3: + self.drop3.addItem(self.drop3_options[drop3_val]) + added_drop3.add(drop3_val) + + if drop4_val in self.drop4_options and drop4_val not in added_drop4: + self.drop4.addItem(self.drop4_options[drop4_val]) + added_drop4.add(drop4_val) + + + def setup_dropdown_grid(self, layout): + grid = QGridLayout() + + labels = [ + ("TRN: Transformations over time", self.drop1), + ("AGG: Aggregation modes", self.drop2), + ("UNIT: Unit codes", self.drop3), + ("REF: Codes for relative performance", self.drop4) + ] + + # Create grid layout for labels and dropdowns + grid = QGridLayout() + for index, (label_text, dropdown) in enumerate(labels): + label = QLabel(label_text) + + # Setting a fixed width ensures all labels have the same width. + # Add some extra whitespaces at end -- for beter alignment. + label.setFixedWidth(label.fontMetrics().width("REF: Codes for relative performance ")) + + grid.addWidget(label, index, 0) + grid.addWidget(dropdown, index, 1) + + layout.addLayout(grid) + + + def setup_action_elements(self, layout, var_name): + self.submit_button = QPushButton("Load Data", self) + self.submit_button.clicked.connect(lambda: self.ameco_final_submit_clicked(var_name)) + # self.submit_button.clicked.connect(self.ameco_final_submit_clicked(var_name)) # check: why only lambda works?? + + # Invalid choice label + self.invalid_label = QLabel("Invalid selection. Please choose a valid combination from the dropdown menus.", self) + self.invalid_label.setStyleSheet("color: red;") + self.invalid_label.hide() # Hide it initially + + layout.addWidget(self.invalid_label) + layout.setAlignment(self.invalid_label, Qt.AlignCenter) + layout.addWidget(self.submit_button) + + # Connect dropdowns to verification function + self.drop1.currentIndexChanged.connect(self.verification_check) + self.drop2.currentIndexChanged.connect(self.verification_check) + self.drop3.currentIndexChanged.connect(self.verification_check) + self.drop4.currentIndexChanged.connect(self.verification_check) + + # Set layout + self.setLayout(layout) + + + def verification_check(self, silent=False): + # At this stage, we have a 'var_name' and 'dropdown selection' provided by user. + # Revert now back to numerical codes so we can subset/load correct larray data. + + # Reverse mapping for dropdowns. Go back from text in dropdowns to numerical codes. + def create_reverse_mapping(original_mapping): + return {v: k for k, v in original_mapping.items()} + + # Apply reverse mappings + rev_drop1_options = create_reverse_mapping(self.drop1_options) + rev_drop2_options = create_reverse_mapping(self.drop2_options) + rev_drop3_options = create_reverse_mapping(self.drop3_options) + rev_drop4_options = create_reverse_mapping(self.drop4_options) + + # Get Selected Options + self.sel1 = rev_drop1_options[self.drop1.currentText()] + self.sel2 = rev_drop2_options[self.drop2.currentText()] + self.sel3 = rev_drop3_options[self.drop3.currentText()] + self.sel4 = rev_drop4_options[self.drop4.currentText()] + + # Merge together 4 dropdown selections to single code (e.g. '1-1-0-0') for subsetting larray + self.numo_code = "-".join([str(self.sel1), str(self.sel2), str(self.sel3), str(self.sel4)]) + + # Extra check if valid choice was made in dropdown combinations. + if [self.sel1, self.sel2, self.sel3, self.sel4] in self.choice_list: + print(f"valid choice: {self.numo_code}") + self.submit_button.show() + self.invalid_label.hide() + return True + else: + print(f"invalid choice: {self.numo_code}") + self.submit_button.hide() + self.invalid_label.show() + if not silent: + QMessageBox.information(self, f"{self.numo_code} is a bad combination.", f"Valid choices are: {self.choice_list}", QMessageBox.Ok) + return False + + def ameco_final_submit_clicked(self, var_name): + if self.verification_check(silent=True): + ameco_subset = self.ameco[self.numo_code, var_name] + + editor = self.parent().parent() + new_data = editor.data.copy() # make a copy of dataset-dictionary before starting to add new datasets + new_data[var_name] = ameco_subset # add dataset for LArrayEditor update + editor.kernel.shell.user_ns[var_name] = ameco_subset # add dataset to console namespace + editor.view_expr(ameco_subset, expr=var_name) + editor.update_mapping(new_data) + + self.accept() + + + +class EurostatBrowserDialog(QDialog): + def __init__(self, index, parent=None): + super(EurostatBrowserDialog, self).__init__(parent) + + assert isinstance(index, pd.DataFrame) + + # drop unused/redundant "type" column + index = index.drop('type', axis=1) + + # display dates using locale + # import locale + # locale.setlocale(locale.LC_ALL, '') + # datetime.date.strftime('%x') + + # CHECK: this builds the whole (model) tree in memory eagerly, so it is not lazy despite using a + # Simple*Lazy*TreeModel, but it is fast enough for now. *If* it ever becomes a problem, + # we could make this lazy pretty easily (see treemodel.LazyDictTreeNode for an example). + root = indented_df_to_treenode(index) + self.df = self.eurostat_tree_to_dataframe(root) + + # Create the tree view UI in Dialog + model = SimpleLazyTreeModel(root) + tree = QTreeView() + tree.setModel(model) + tree.setUniformRowHeights(True) + tree.doubleClicked.connect(self.view_eurostat_indicator) + tree.setColumnWidth(0, 320) + tree.setContextMenuPolicy(Qt.CustomContextMenu) + tree.customContextMenuRequested.connect(self.show_context_menu) + self.tree = tree + + # Create the search results list (and hide as default) + self.search_results_list = QListWidget() + self.search_results_list.itemClicked.connect(self.handle_search_item_click) + self.search_results_list.hide() + + # Create the search input field + self.search_input = QLineEdit(self) + self.search_input.setPlaceholderText("Search...") + self.search_input.textChanged.connect(self.handle_search) + + # Create the "Advanced Import" button + self.advanced_button = QPushButton("Import from Configuration File", self) + self.advanced_button.setFocusPolicy(Qt.NoFocus) # turn off + self.advanced_button.clicked.connect(self.showAdvancedPopup) + + # General settings: resize + title + self.resize(950, 600) + self.setWindowTitle("Select dataset") + + # Add widgets to layout + layout = QVBoxLayout() + layout.addWidget(self.search_input) + layout.addWidget(tree) + layout.addWidget(self.search_results_list) + layout.addWidget(self.advanced_button) + self.setLayout(layout) + + + def eurostat_tree_to_dataframe(self, root): + # Recursively traverse tree and extract data *only* of leaf nodes (into dataframe) + def traverse_tree(node, rows): + if not node.children: # If the node has no children, append its data to rows + rows.append(node.data) + for child in node.children: # If the node has children, recursively call the function for each child + traverse_tree(child, rows) + rows = [] + traverse_tree(root, rows) + return pd.DataFrame(rows, columns=root.data) + + + def show_context_menu(self, position): + menu = QMenu(self) + + # Only show the "Add to list" UI element if the current node does not have children + index = self.tree.currentIndex() + node = index.internalPointer() + if not node.children: + add_to_list_action = QAction("Add to list", self) + add_to_list_action.triggered.connect(self.add_to_list) + menu.addAction(add_to_list_action) + + # Display the context menu at the position of the mouse click + global_position = self.tree.viewport().mapToGlobal(position) + menu.exec_(global_position) + + + def add_to_list(self): + index = self.tree.currentIndex() + node = index.internalPointer() + + if not node.children: + # custom tree model where each node was represented by tuple (name, code,...) => data[1] = code + code = self.tree.currentIndex().internalPointer().data[1] + arr = eurostat_get(code, cache_dir='__array_cache__') + + # Add indicator to the list + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + editor.kernel.shell.user_ns[code] = arr + + + def view_eurostat_indicator(self, index): + node = index.internalPointer() + title, code, last_update_of_data, last_table_structure_change, data_start, data_end = node.data + if not node.children: + last_update_of_data = datetime.strptime(last_update_of_data, "%d.%m.%Y") + last_table_structure_change = datetime.strptime(last_table_structure_change, "%d.%m.%Y") + last_change = max(last_update_of_data, last_table_structure_change) + try: + arr = eurostat_get(code, maxage=last_change, cache_dir='__array_cache__') + current_dataset_labels = arr.freq.labels + # Present frequency popup only if there are multiple frequencies + if len(current_dataset_labels) > 1: + dialog = FrequencyFilterDialog(current_dataset_labels, self) # first argument so that only relevant labels appear in popup + result = dialog.exec_() + if result == QDialog.Accepted: + selected_frequencies = dialog.get_selected_frequencies() + else: + selected_frequencies = current_dataset_labels + + arr = freq_eurostat_editor(selected_frequencies, arr) + + # Load the data into the editor + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + self.parent().kernel.shell.user_ns[code] = arr + self.accept() + except Exception: + QMessageBox.critical(self, "Error", "Failed to load {}".format(code)) + + + def keyPressEvent(self, event): + if event.key() in [Qt.Key_Return, Qt.Key_Enter] and self.tree.hasFocus(): + self.view_eurostat_indicator(self.tree.currentIndex()) + super(EurostatBrowserDialog, self).keyPressEvent(event) + + + def handle_search(self, text): + if text: + # when text is entered, then hide the treeview and show (new) search results + self.tree.hide() + self.search_results_list.clear() + + # filter dataframe based on search term(s) + filtered_df = multi_search_df(self.df, text) + + # drop duplicates in search result (i.e. same code, originating from different locations tree) + filtered_df = filtered_df.drop_duplicates(subset='Code') + + # reduce dataframe to list of strings (only retain 'title' and 'code') + results = [f"{row['Title']} ({row['Code']})" for _, row in filtered_df.iterrows()] + + self.search_results_list.addItems(results) + self.search_results_list.show() + + else: # if search field is empty + self.tree.show() + self.search_results_list.hide() + + + def handle_search_item_click(self, item): + # Extract all relevant text within last parentheses; BUT only need last parentheses, cf. "my country (Belgium) subject X (code)" + # Assumes the (code) is always contained in last parentheses + matches = re.findall(r'\(([^)]+)\)', item.text()) + + if matches: + last_match = matches[-1] + + # Use the code extracted text from the clicked item (this ought to be correct dataset code) + code = last_match + + # Use 'code' to load the dataset via eurostat_get + from larray_eurostat import eurostat_get + arr = eurostat_get(code, cache_dir='__array_cache__') + + # Add loaded dataset to editor and shell + editor = self.parent() + new_data = editor.data.copy() + new_data[code] = arr + editor.update_mapping(new_data) + editor.kernel.shell.user_ns[code] = arr + + + def showAdvancedPopup(self): + popup = AdvancedPopup(self) + # Next Line takes into account 'finished signal' of the AdvancedPopup instance and links it to closeDialog. + # In other words: closing the advancedpopup dialog box shall also close the primary dialog box. + popup.finished.connect(self.closeDialog) + popup.exec_() + + + def closeDialog(self): + self.close() + + + +class AdvancedPopup(QDialog): + def __init__(self, parent=None): + # Self-commentary: an additional/extra parent argument with default value None is added. + # This subtletly relates to following line of code elsewhere: + # >> popup = AdvancedPopup(self) + # The explicit 'self' argument here corresponds to the second 'parent' of AdvancedPopup (not the implicit first 'self' to be used inside the class) + # Put differently, the 'self' written in the expression 'AdvancedPopup(self)' is *at the location of that expression* actually referring to EurostatBrowserDialog + + # Pass the parent to the base class constructor + super().__init__(parent) + + # Inititalize the UI and set default values + self.initUI() + self.yaml_content = {} + + + def initUI(self): + self.resize(600, 600) + layout = QVBoxLayout(self) + + # Button to load YAML + self.loadButton = QPushButton("Load YAML", self) + self.loadButton.clicked.connect(self.loadYAML) + layout.addWidget(self.loadButton) + + # TextEdit for YAML content + self.yamlTextEdit = QTextEdit(self) + + import textwrap + default_yaml = textwrap.dedent("""\ + # First indicator with its specific settings. + indicator_name_1: + name: var1, var2, var3 + var1: + label1: renamed_label1 + label2: renamed_label2 + var2: + label1: renamed_label1 + label2: renamed_label2 + ... + # You can continue adding more indicators and labels as per your requirements. + """) + + # Set default YAML content (example) + self.yamlTextEdit.setPlainText(default_yaml) + layout.addWidget(self.yamlTextEdit) + + # Checkbox for generating a copy + self.generateCopyCheckBox = QCheckBox("Generate additional copy", self) + layout.addWidget(self.generateCopyCheckBox) + + # Horizontal layout for dropdown and path selection + copyLayout = QHBoxLayout() + + # ComboBox (Drop-down) for file formats + self.fileFormatComboBox = QComboBox(self) + self.fileFormatComboBox.addItems(["xlsx", "IODE", "csv"]) + copyLayout.addWidget(self.fileFormatComboBox) + + # Button for path selection + self.pathSelectionButton = QPushButton("Select Path...", self) + self.pathSelectionButton.clicked.connect(self.selectPath) + copyLayout.addWidget(self.pathSelectionButton) + layout.addLayout(copyLayout) + + # Final button to proceed with the main task and optionally generate a copy + self.proceedButton = QPushButton("Proceed", self) + self.proceedButton.clicked.connect(self.proceedWithTasks) + layout.addWidget(self.proceedButton) + + + def loadYAML(self): + options = QFileDialog.Options() + filePath, _ = QFileDialog.getOpenFileName(self, "Open YAML File", "", "YAML Files (*.yaml *.yml);;All Files (*)", options=options) + if filePath: + with open(filePath, 'r') as file: + self.yaml_content = yaml.safe_load(file) + + # don't show dbdir and format in the textbox, they are already shown in the UI + self.yaml_content_subset = {k: v for k, v in self.yaml_content.items() if k not in ['dbdir', 'format']} + self.yamlTextEdit.setText(yaml.dump(self.yaml_content_subset, default_flow_style=False)) + + # Check for 'dbdir' and 'format' in the loaded content + if 'dbdir' in self.yaml_content and 'format' in self.yaml_content: + format_val = self.yaml_content['format'] + self.setComboBoxValue(format_val) + + + def setComboBoxValue(self, value): + index = self.fileFormatComboBox.findText(value) + if index != -1: # if the format from YAML exists in the ComboBox items + self.fileFormatComboBox.setCurrentIndex(index) + + + def selectPath(self): + options = QFileDialog.Options() + import os + if self.yaml_content is not None and 'dbdir' in self.yaml_content and self.yaml_content['dbdir'] is not None and os.path.exists(self.yaml_content['dbdir']) and os.path.isdir(self.yaml_content['dbdir']): + # if appropriate, then use the dbdir from the loaded YAML file as the default path in QFileDialog + self.selectedPath, _ = QFileDialog.getSaveFileName(self, "Select Path", self.yaml_content['dbdir'], "All Files (*)", options=options) + else: + self.selectedPath, _ = QFileDialog.getSaveFileName(self, "Select Path", "", "All Files (*)", options=options) + + + def proceedWithTasks(self): + # Preliminary functions to extract the relevant data + def intersect(a, b): + if isinstance(a, str): + a = a.split(',') + elif isinstance(a, dict): + a = a.keys() + return [val for val in a if val in b] + + def extract_mat(mat, keys): + intersect_keys = {} + replace_keys = {} + name_keys = [] + new_keys = [] + none_keys = [] + + for k in keys: + # name: defines the order and the naming convention + if k == 'name': + name_keys = keys[k].split(',') + new_keys = [n for n in name_keys if n not in mat.axes.names] + + # select common labels + else: + intersect_keys[k] = intersect(keys[k], mat.axes[k].labels) + # prepare dict for replacement + if isinstance(keys[k], dict): + replace_keys[k] = {key: keys[k][key] for key in intersect_keys[k]} + for key, value in replace_keys[k].items(): + if value is None: + replace_keys[k][key] = key + none_keys.append(key) + + mat = mat[intersect_keys] + # replace labels + if len(replace_keys) > 0: + mat = mat.set_labels(replace_keys) + + if len(none_keys) > 0: + for nk in none_keys: + mat = mat[nk] + + # add missing dimensions + if len(new_keys) > 0: + for nk in new_keys: + mat = la.stack({nk: mat}, nk) + # put in correct order + if len(name_keys) > 0: + mat = mat.transpose(name_keys) + return mat + + if self.generateCopyCheckBox.isChecked(): + # Use UI choice as most recent info (instead of original YAML file stored in pm['dbdir'] etc.) + # Reasoning/philosophy: load YAML file, maybe later use dropdown to export other formats. + file_format = self.fileFormatComboBox.currentText() + db_dir = self.yaml_content['dbdir'] + + # There might be multiple datasets in YAML config. Redundancy to load them *all* in viewer sequentially + # Cannot see them all. So, convert dictionary to list: easier to navigate to last one (only load this). + # Note: two different tasks: 1) add datasets to Editor as available larray objects, and b) view those + # graphically in UI. Only makes sense to 'view' the last dataset if multiple indicators are in YAML, but + # all of the datasets in the YAML needed to be added to session/editor regardless. + + # Prepare list for easy access + pm = self.yaml_content + indicators_as_list = list(pm['indicators']) + + # Prepare access to already existing datasets in editor + editor = self.parent().parent() # need to be *two* class-levels higher + new_data = editor.data.copy() # make a copy of dataset-dictionary before starting to add new datasets + + for code in indicators_as_list: + # Adding datasets + arr = eurostat_get(code, cache_dir='__array_cache__') # pulls dataset + arr = extract_mat(arr, pm['indicators'][code]) # relabels dataset via YAML configs + + new_data[code] = arr # add dataset for LArrayEditor update + editor.kernel.shell.user_ns[code] = arr # add dataset to console namespace + + # Viewing datasets: only if last indicator in YAML config + if code == indicators_as_list[-1]: + editor.view_expr(arr, expr=code) + + # Export for different file format (...) + if self.generateCopyCheckBox.isChecked(): + if file_format == 'csv': + arr.to_csv(f'{db_dir}/{code}.csv') + elif file_format == 'ui': + # la.view(s) + pass + elif file_format in ['var', 'iode']: + # to_var(arr, db_dir) + pass + elif file_format[:3] == 'xls': + arr.to_excel(f'{db_dir}/{code}.xlsx') + else: + arr.to_hdf(f'{db_dir}/{code}') + + # Update mapping outside for loop -- i.e. no need to do this update multiple times + editor.update_mapping(new_data) + self.accept() + + + class AbstractEditor(QMainWindow): """Abstract Editor Window""" @@ -553,6 +1469,9 @@ def _setup_file_menu(self, menu_bar): # ============= # file_menu.addSeparator() file_menu.addAction(create_action(self, _('&Load Example Dataset'), triggered=self.load_example)) + file_menu.addAction(create_action(self, _('&Browse Eurostat Datasets'), triggered=self.browse_eurostat)) + file_menu.addAction(create_action(self, _('&Browse AMECO Datasets'), triggered=self.browse_ameco)) + # ============= # # SCRIPTS # # ============= # @@ -675,7 +1594,7 @@ def line_edit_update(self): def view_expr(self, array, expr): self._listwidget.clearSelection() - self.set_current_array(array, expr) + self.set_current_array(array, name=expr) def _display_in_grid(self, k, v): return not k.startswith('__') and isinstance(v, DISPLAY_IN_GRID) @@ -1153,6 +2072,23 @@ def load_example(self): filepath = AVAILABLE_EXAMPLE_DATA[dataset_name] self._open_file(filepath) + def browse_eurostat(self): + from larray_eurostat import get_index + try: + df = get_index(cache_dir='__array_cache__', maxage=None) + except: + QMessageBox.critical(self, "Error", "Failed to fetch Eurostat dataset index") + return + dialog = EurostatBrowserDialog(df, parent=self) + dialog.show() + + def browse_ameco(self): + with open('larray_editor/ameco_toc_tree.txt', 'r', encoding='utf-8') as file: + ameco_table_of_contents = file.read() + + dialog = AmecoBrowserDialog(ameco_table_of_contents, parent=self) + dialog.show() + class ArrayEditor(AbstractEditor): """Array Editor Dialog""" diff --git a/larray_editor/treemodel.py b/larray_editor/treemodel.py new file mode 100644 index 0000000..2b4a7f3 --- /dev/null +++ b/larray_editor/treemodel.py @@ -0,0 +1,110 @@ +from qtpy.QtCore import Qt, QModelIndex, QAbstractItemModel + + +class SimpleTreeNode(object): + __slots__ = ['parent', 'data', 'children'] + + def __init__(self, parent, data): + self.parent = parent + self.data = data + self.children = [] + + +class SimpleLazyTreeModel(QAbstractItemModel): + def __init__(self, root, parent=None): + super(SimpleLazyTreeModel, self).__init__(parent) + assert isinstance(root, SimpleTreeNode) + self.root = root + + def columnCount(self, index): + node = index.internalPointer() if index.isValid() else self.root + return len(node.data) + + def data(self, index, role): + if not index.isValid(): + return None + + if role != Qt.DisplayRole: + return None + + node = index.internalPointer() + return node.data[index.column()] + + def flags(self, index): + if not index.isValid(): + return Qt.NoItemFlags + + return Qt.ItemIsEnabled | Qt.ItemIsSelectable + + def headerData(self, section, orientation, role): + if orientation == Qt.Horizontal and role == Qt.DisplayRole: + return self.root.data[section] + return None + + def index(self, row, column, parent_index): + if not self.hasIndex(row, column, parent_index): + return QModelIndex() + + parent_node = parent_index.internalPointer() if parent_index.isValid() else self.root + child_node = parent_node.children[row] + return self.createIndex(row, column, child_node) + + def parent(self, index): + if not index.isValid(): + return QModelIndex() + + child_node = index.internalPointer() + parent_node = child_node.parent + + if parent_node == self.root: + return QModelIndex() + + grand_parent = parent_node.parent + parent_row = grand_parent.children.index(parent_node) + return self.createIndex(parent_row, 0, parent_node) + + def rowCount(self, index): + if index.column() > 0: + return 0 + + node = index.internalPointer() if index.isValid() else self.root + return len(node.children) + + +## Extra functions for AMECO tree +class Ameco_SimpleTreeNode: + def __init__(self, data, parent=None): + self.data = data + self.parent = parent + self.children = [] + if parent: + parent.children.append(self) + +def Ameco_parse_tree_structure(lines): + root = Ameco_SimpleTreeNode("Root") + prev_node = root + prev_level = -1 + + for line in lines: + level = line.count('>') + name = line.strip('>').strip() + node = Ameco_SimpleTreeNode(name) + + if level > prev_level: + node.parent = prev_node + prev_node.children.append(node) + elif level == prev_level: + node.parent = prev_node.parent + prev_node.parent.children.append(node) + else: + diff = prev_level - level + higher_node = prev_node.parent + for _ in range(diff): + higher_node = higher_node.parent + node.parent = higher_node + higher_node.children.append(node) + + prev_node = node + prev_level = level + + return root \ No newline at end of file