Coverage for mlair/helpers/data_sources/toar_data.py: 66%

1__author__ = "Lukas Leufen" 

2__date__ = "2022-07-05" 

3 

4from . import join, toar_data_v2 

5 

6import requests 

7import pandas as pd 

8from .data_loader import EmptyQueryResult 

9import xarray as xr 

10 

11 

12def download_toar(station, toar_stats, sampling, data_origin, window_dim, time_dim, target_dim): 

13 try: 

14 # load data from toar-data (v2) 

15 df_toar, meta_toar = toar_data_v2.download_toar(station, toar_stats, sampling=sampling, data_origin=data_origin) 

16 except (AttributeError, EmptyQueryResult, KeyError, requests.ConnectionError, ValueError, IndexError): 

17 df_toar, meta_toar = None, None 

18 

19 try: 

20 # load join data (toar-data v1) 

21 df_join, meta_join = join.download_join(station_name=station, stat_var=toar_stats, sampling=sampling, 

22 data_origin=data_origin) 

23 except (AttributeError, EmptyQueryResult, KeyError, requests.ConnectionError, ValueError, IndexError): 

24 df_join, meta_join = None, None 

25 

26 # merge both data sources with priority on toar-data v2 

27 if df_toar is not None and df_join is not None: 27 ↛ 28line 27 didn't jump to line 28, because the condition on line 27 was never true

28 df_merged = merge_toar_join(df_toar, df_join, sampling) 

29 meta_merged = meta_toar 

30 else: 

31 df_merged = df_toar if df_toar is not None else df_join 

32 meta_merged = meta_toar if df_toar is not None else meta_join 

33 xarr_merged = xr.DataArray(df_merged, dims=[time_dim, target_dim]) 

34 xarr_merged = xarr_merged.expand_dims({window_dim: [0]}) 

35 return xarr_merged, meta_merged 

36 

37 

38def merge_toar_join(df_toar, df_join, sampling): 

39 start_date = min([df_toar.index.min(), df_join.index.min()]) 

40 end_date = max([df_toar.index.max(), df_join.index.max()]) 

41 freq = {"hourly": "1H", "daily": "1d"}.get(sampling) 

42 full_time = pd.date_range(start_date, end_date, freq=freq) 

43 full_data = df_toar.reindex(full_time) 

44 full_data.update(df_join, overwrite=False) 

45 return full_data