File size: 3,809 Bytes
98024ab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | import re
import math
import numpy as np
import pandas as pd
DEFAULT_WEATHER_AGG_RULES = {
"dayl": "mean",
"prcp": "sum",
"srad": "mean",
"tmax": "mean",
"tmin": "mean",
"vp": "mean",
"tmean": "mean",
"gdd": "sum",
"precip_3day_avg_perday": "mean",
"precip_7day_avg_perday": "mean",
"precip_14day_avg_perday": "mean",
}
def load_table(path):
path = str(path)
if path.endswith(".h5") or path.endswith(".hdf5"):
return pd.read_hdf(path)
if path.endswith(".csv"):
return pd.read_csv(path)
if path.endswith(".parquet"):
return pd.read_parquet(path)
raise ValueError(f"Unsupported file type: {path}")
def decode_bytes_in_object_cols(df: pd.DataFrame) -> pd.DataFrame:
out = df.copy()
for c in out.select_dtypes(include=["object"]).columns:
if len(out) > 0 and isinstance(out[c].iloc[0], (bytes, bytearray)):
out[c] = out[c].str.decode("utf-8")
return out
def find_daily_cols(df: pd.DataFrame, var: str) -> list[str]:
pat_bracket = re.compile(rf"^{re.escape(var)}.*?_\[(\d+)\]$")
pat_plain = re.compile(rf"^{re.escape(var)}.*?_(\d+)$")
hits = []
for c in df.columns:
s = str(c)
m = pat_bracket.match(s) or pat_plain.match(s)
if m:
hits.append((int(m.group(1)), c))
hits.sort(key=lambda x: x[0])
return [c for _, c in hits]
# def daily_to_cumulative_weekly(daily: np.ndarray, agg: str, week_len: int = 7) -> np.ndarray:
# daily = daily.astype(np.float32)
# T = daily.shape[0]
# K = int(math.ceil(T / week_len))
# out = np.zeros(K, dtype=np.float32)
# for w in range(K):
# e = min((w + 1) * week_len, T)
# chunk = daily[:e]
# out[w] = np.nansum(chunk) if agg == "sum" else np.nanmean(chunk)
# return out
def daily_to_cumulative_weekly(
daily: np.ndarray,
agg: str,
week_len: int = 7,
) -> np.ndarray:
daily = daily.astype(np.float32)
T = daily.shape[0]
K = int(math.ceil(T / week_len))
weekly = np.zeros(K, dtype=np.float32)
cumulative = np.zeros(K, dtype=np.float32)
for w in range(K):
s = w * week_len
e = min((w + 1) * week_len, T)
week_chunk = daily[s:e]
cumulative_chunk = daily[:e]
if agg == "sum":
weekly[w] = np.nansum(week_chunk)
cumulative[w] = np.nansum(cumulative_chunk)
else:
weekly[w] = np.nanmean(week_chunk)
cumulative[w] = np.nanmean(cumulative_chunk)
return np.stack([weekly, cumulative], axis=1).astype(np.float32)
def compute_x_stats(dataset, max_samples=20000, seed=0):
rng = np.random.default_rng(seed)
n = min(len(dataset), max_samples)
idxs = rng.choice(len(dataset), size=n, replace=False) if n < len(dataset) else np.arange(len(dataset))
weather, soil = [], []
for i in idxs:
item = dataset[int(i)]
weather.append(item["weather"].numpy())
soil.append(item["soil"].numpy())
W_all = np.stack(weather).astype(np.float32)
S_all = np.stack(soil).astype(np.float32)
w_mean = np.nanmean(W_all, axis=(0, 1))
w_std = np.nanstd(W_all, axis=(0, 1)) + 1e-6
s_mean = np.nanmean(S_all, axis=0)
s_std = np.nanstd(S_all, axis=0) + 1e-6
w_mean = np.where(np.isfinite(w_mean), w_mean, 0.0)
w_std = np.where((np.isfinite(w_std)) & (w_std > 1e-6), w_std, 1.0)
s_mean = np.where(np.isfinite(s_mean), s_mean, 0.0)
s_std = np.where((np.isfinite(s_std)) & (s_std > 1e-6), s_std, 1.0)
return w_mean, w_std, s_mean, s_std
def compute_y_stats(dataset):
ys = np.array([float(dataset[i]["yield"]) for i in range(len(dataset))], dtype=np.float32)
#ys_log = np.log1p(np.clip(ys, 0.0, None))
return float(ys.mean()), float(ys.std() + 1e-6) |