diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-map.png new file mode 100644 index 0000000..4281d50 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-sizes.png new file mode 100644 index 0000000..b6a8824 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-pairs.png new file mode 100644 index 0000000..8b1d63d Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-projection.png new file mode 100644 index 0000000..544cef2 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-1-effective-1/location-1__run-1__farm__k-1__effective-1__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-map.png new file mode 100644 index 0000000..5bfa8e3 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-sizes.png new file mode 100644 index 0000000..8932e9b Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-pairs.png new file mode 100644 index 0000000..f99b8a1 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-projection.png new file mode 100644 index 0000000..db8d780 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-10-effective-10/location-1__run-1__farm__k-10__effective-10__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-map.png new file mode 100644 index 0000000..6848735 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-sizes.png new file mode 100644 index 0000000..09a2166 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-pairs.png new file mode 100644 index 0000000..86da49e Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-projection.png new file mode 100644 index 0000000..8dba4c6 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-2-effective-2/location-1__run-1__farm__k-2__effective-2__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-map.png new file mode 100644 index 0000000..0262e88 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-sizes.png new file mode 100644 index 0000000..0917f83 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-pairs.png new file mode 100644 index 0000000..f8ee15b Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-projection.png new file mode 100644 index 0000000..e7d7e82 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-3-effective-3/location-1__run-1__farm__k-3__effective-3__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-map.png new file mode 100644 index 0000000..5978b64 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-sizes.png new file mode 100644 index 0000000..909bb07 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-pairs.png new file mode 100644 index 0000000..da370ef Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-projection.png new file mode 100644 index 0000000..d2026fc Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-4-effective-4/location-1__run-1__farm__k-4__effective-4__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-map.png new file mode 100644 index 0000000..b7e9a2a Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-sizes.png new file mode 100644 index 0000000..76eee68 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-pairs.png new file mode 100644 index 0000000..04b7fa4 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-projection.png new file mode 100644 index 0000000..ac588e8 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-5-effective-5/location-1__run-1__farm__k-5__effective-5__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-map.png new file mode 100644 index 0000000..ef25e99 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-sizes.png new file mode 100644 index 0000000..82439f6 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-pairs.png new file mode 100644 index 0000000..1321428 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-projection.png new file mode 100644 index 0000000..53a35b8 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-6-effective-6/location-1__run-1__farm__k-6__effective-6__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-map.png new file mode 100644 index 0000000..5d0edbc Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-sizes.png new file mode 100644 index 0000000..ec4570a Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-pairs.png new file mode 100644 index 0000000..88d9d88 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-projection.png new file mode 100644 index 0000000..720800d Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-7-effective-7/location-1__run-1__farm__k-7__effective-7__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-map.png new file mode 100644 index 0000000..e293876 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-sizes.png new file mode 100644 index 0000000..834f7e2 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-pairs.png new file mode 100644 index 0000000..79964a3 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-projection.png new file mode 100644 index 0000000..10ffcfc Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-8-effective-8/location-1__run-1__farm__k-8__effective-8__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-map.png new file mode 100644 index 0000000..faea4ad Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-sizes.png new file mode 100644 index 0000000..d78f8f0 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__elbow.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__elbow.png new file mode 100644 index 0000000..a51a107 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__elbow.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-pairs.png new file mode 100644 index 0000000..85ec61e Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-projection.png new file mode 100644 index 0000000..756daf4 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/k-9-effective-9/location-1__run-1__farm__k-9__effective-9__feature-projection.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-map.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-map.png index c2e315a..6848735 100644 Binary files a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-map.png and b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-map.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-sizes.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-sizes.png index 2089897..09a2166 100644 Binary files a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-sizes.png and b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__cluster-sizes.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-pairs.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-pairs.png index 3dacfd6..86da49e 100644 Binary files a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-pairs.png and b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-pairs.png differ diff --git a/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-projection.png b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-projection.png new file mode 100644 index 0000000..8dba4c6 Binary files /dev/null and b/artifacts/remote_sensing_charts/location-1/run-1-farm/location-1__run-1__farm__feature-projection.png differ diff --git a/location_data/admin.py b/location_data/admin.py index c305966..269e70c 100644 --- a/location_data/admin.py +++ b/location_data/admin.py @@ -3,7 +3,11 @@ from .models import ( AnalysisGridCell, AnalysisGridObservation, BlockSubdivision, + RemoteSensingClusterBlock, RemoteSensingClusterAssignment, + RemoteSensingSubdivisionOption, + RemoteSensingSubdivisionOptionAssignment, + RemoteSensingSubdivisionOptionBlock, RemoteSensingRun, RemoteSensingSubdivisionResult, SoilLocation, @@ -121,6 +125,68 @@ class RemoteSensingSubdivisionResultAdmin(admin.ModelAdmin): readonly_fields = ("created_at", "updated_at") +@admin.register(RemoteSensingClusterBlock) +class RemoteSensingClusterBlockAdmin(admin.ModelAdmin): + list_display = ( + "uuid", + "soil_location", + "block_code", + "sub_block_code", + "cluster_label", + "cell_count", + "updated_at", + ) + list_filter = ("cluster_label", "chunk_size_sqm", "created_at") + search_fields = ("uuid", "block_code", "sub_block_code", "soil_location__latitude", "soil_location__longitude") + readonly_fields = ("created_at", "updated_at") + + +@admin.register(RemoteSensingSubdivisionOption) +class RemoteSensingSubdivisionOptionAdmin(admin.ModelAdmin): + list_display = ( + "id", + "result", + "requested_k", + "effective_cluster_count", + "is_active", + "is_recommended", + "selection_source", + "updated_at", + ) + list_filter = ("is_active", "is_recommended", "selection_source", "created_at") + search_fields = ("result__block_code", "result__soil_location__latitude", "result__soil_location__longitude") + readonly_fields = ("created_at", "updated_at") + + +@admin.register(RemoteSensingSubdivisionOptionBlock) +class RemoteSensingSubdivisionOptionBlockAdmin(admin.ModelAdmin): + list_display = ( + "id", + "option", + "cluster_label", + "sub_block_code", + "cell_count", + "updated_at", + ) + list_filter = ("cluster_label", "created_at") + search_fields = ("sub_block_code", "option__result__block_code") + readonly_fields = ("created_at", "updated_at") + + +@admin.register(RemoteSensingSubdivisionOptionAssignment) +class RemoteSensingSubdivisionOptionAssignmentAdmin(admin.ModelAdmin): + list_display = ( + "id", + "option", + "cell", + "cluster_label", + "updated_at", + ) + list_filter = ("cluster_label", "created_at") + search_fields = ("cell__cell_code", "option__result__block_code") + readonly_fields = ("created_at", "updated_at") + + @admin.register(RemoteSensingClusterAssignment) class RemoteSensingClusterAssignmentAdmin(admin.ModelAdmin): list_display = ( diff --git a/location_data/data_driven_subdivision.py b/location_data/data_driven_subdivision.py index b688da4..1228a03 100644 --- a/location_data/data_driven_subdivision.py +++ b/location_data/data_driven_subdivision.py @@ -5,6 +5,7 @@ import math import os from pathlib import Path from dataclasses import dataclass +from decimal import Decimal import json import logging from typing import Any @@ -13,13 +14,17 @@ from django.conf import settings from django.core.files.base import ContentFile from django.db import transaction -from .block_subdivision import detect_elbow_point, render_elbow_plot +from .block_subdivision import detect_elbow_point, point_in_polygon, render_elbow_plot from .models import ( AnalysisGridObservation, BlockSubdivision, + RemoteSensingClusterBlock, RemoteSensingClusterAssignment, RemoteSensingRun, RemoteSensingSubdivisionResult, + RemoteSensingSubdivisionOption, + RemoteSensingSubdivisionOptionAssignment, + RemoteSensingSubdivisionOptionBlock, SoilLocation, ) @@ -62,6 +67,17 @@ class ClusteringDataset: skipped_reasons: dict[str, list[str]] +@dataclass +class SubdivisionOptionPayload: + requested_k: int + effective_cluster_count: int + labels: list[int] + cluster_summaries: list[dict[str, Any]] + spatial_constraint_metadata: dict[str, Any] + assignment_rows: list[dict[str, Any]] + cluster_block_rows: list[dict[str, Any]] + + def create_remote_sensing_subdivision_result( *, location: SoilLocation, @@ -95,15 +111,28 @@ def create_remote_sensing_subdivision_result( random_state=random_state, ) cluster_selection_strategy = "explicit_k" if explicit_k is not None else "elbow" - labels = run_kmeans_labels( - scaled_matrix=dataset.scaled_matrix, - cluster_count=optimal_k, + option_payloads = build_subdivision_option_payloads( + dataset=dataset, + max_k=max_k, random_state=random_state, ) - cluster_summaries = build_cluster_summaries( - observations=dataset.observations, - labels=labels, + if not option_payloads: + raise DataDrivenSubdivisionError("هیچ گزینه K معتبری برای ذخیره‌سازی subdivision ساخته نشد.") + + active_requested_k = min( + int(explicit_k if explicit_k is not None else optimal_k), + max(option_payload.requested_k for option_payload in option_payloads), ) + active_option_payload = next( + ( + option_payload + for option_payload in option_payloads + if option_payload.requested_k == active_requested_k + ), + option_payloads[-1], + ) + effective_cluster_count = active_option_payload.effective_cluster_count + cluster_summaries = active_option_payload.cluster_summaries with transaction.atomic(): result, _created = RemoteSensingSubdivisionResult.objects.update_or_create( @@ -115,7 +144,7 @@ def create_remote_sensing_subdivision_result( "chunk_size_sqm": run.chunk_size_sqm, "temporal_start": run.temporal_start, "temporal_end": run.temporal_end, - "cluster_count": optimal_k, + "cluster_count": effective_cluster_count, "selected_features": dataset.selected_features, "skipped_cell_codes": dataset.skipped_cell_codes, "metadata": { @@ -134,36 +163,61 @@ def create_remote_sensing_subdivision_result( "random_state": random_state, "explicit_k": explicit_k, "selected_k": optimal_k, + "recommended_k": optimal_k, + "active_requested_k": active_requested_k, + "effective_k": effective_cluster_count, "max_k": max_k, "n_init": 10, "selection_strategy": cluster_selection_strategy, }, + "recommended_requested_k": optimal_k, + "active_requested_k": active_requested_k, + "available_k_options": [ + { + "requested_k": option_payload.requested_k, + "effective_cluster_count": option_payload.effective_cluster_count, + } + for option_payload in option_payloads + ], + "spatial_constraint": active_option_payload.spatial_constraint_metadata, "inertia_curve": inertia_curve, "cluster_summaries": cluster_summaries, }, }, ) result.assignments.all().delete() - assignment_rows = [] - for index, observation in enumerate(dataset.observations): - assignment_rows.append( - RemoteSensingClusterAssignment( - result=result, - cell=observation.cell, - cluster_label=int(labels[index]), - raw_feature_values=dataset.raw_feature_maps[index], - scaled_feature_values={ - feature_name: round(dataset.scaled_matrix[index][feature_index], 6) - for feature_index, feature_name in enumerate(dataset.selected_features) - }, - ) - ) - RemoteSensingClusterAssignment.objects.bulk_create(assignment_rows) + result.cluster_blocks.all().delete() + result.options.all().delete() + option_objects = persist_subdivision_options( + result=result, + location=location, + block_subdivision=block_subdivision, + option_payloads=option_payloads, + recommended_requested_k=optimal_k, + active_requested_k=active_requested_k, + chunk_size_sqm=run.chunk_size_sqm, + ) + persist_subdivision_option_artifacts( + result=result, + option_payloads=option_payloads, + option_objects=option_objects, + observations=dataset.observations, + selected_features=dataset.selected_features, + scaled_matrix=dataset.scaled_matrix, + inertia_curve=inertia_curve, + ) + active_option = option_objects[active_requested_k] + activate_subdivision_option( + option=active_option, + selection_source="system", + recommended_requested_k=optimal_k, + ) + result.refresh_from_db() diagnostic_artifacts = _persist_remote_sensing_diagnostic_artifacts( result=result, observations=dataset.observations, - labels=labels, - cluster_summaries=cluster_summaries, + labels=active_option_payload.labels, + cluster_summaries=active_option_payload.cluster_summaries, selected_features=dataset.selected_features, scaled_matrix=dataset.scaled_matrix, inertia_curve=inertia_curve, @@ -173,18 +227,6 @@ def create_remote_sensing_subdivision_result( metadata["diagnostic_artifacts"] = diagnostic_artifacts result.metadata = metadata result.save(update_fields=["metadata", "updated_at"]) - if block_subdivision is not None: - sync_block_subdivision_with_result( - block_subdivision=block_subdivision, - result=result, - observations=observations, - cluster_summaries=cluster_summaries, - ) - sync_location_block_layout_with_result( - location=location, - result=result, - cluster_summaries=cluster_summaries, - ) return result @@ -381,6 +423,77 @@ def run_kmeans_labels( return [int(label) for label in model.fit_predict(scaled_matrix)] +def enforce_spatial_contiguity( + *, + observations: list[AnalysisGridObservation], + labels: list[int], + scaled_matrix: list[list[float]], +) -> tuple[list[int], dict[str, Any]]: + if not observations or len(observations) <= 1: + return labels, { + "applied": False, + "strategy": "shared_edge_component_merge", + "initial_cluster_count": len(set(labels)), + "final_cluster_count": len(set(labels)), + "disconnected_components_merged": 0, + "shared_border_required": True, + } + + adjacency_map = _build_shared_border_adjacency(observations) + if not adjacency_map: + return labels, { + "applied": False, + "strategy": "shared_edge_component_merge", + "initial_cluster_count": len(set(labels)), + "final_cluster_count": len(set(labels)), + "disconnected_components_merged": 0, + "shared_border_required": True, + "note": "No shared-border adjacency detected.", + } + + working_labels = [int(label) for label in labels] + merged_component_count = 0 + max_iterations = max(len(observations), 1) + + for _iteration in range(max_iterations): + disconnected_components = _find_disconnected_label_components( + labels=working_labels, + adjacency_map=adjacency_map, + ) + if not disconnected_components: + normalized_labels = _normalize_cluster_labels(working_labels) + return normalized_labels, { + "applied": merged_component_count > 0, + "strategy": "shared_edge_component_merge", + "initial_cluster_count": len(set(labels)), + "final_cluster_count": len(set(normalized_labels)), + "disconnected_components_merged": merged_component_count, + "shared_border_required": True, + } + + for disconnected_component in disconnected_components: + target_label = _choose_neighbor_label_for_component( + component_indexes=disconnected_component, + labels=working_labels, + adjacency_map=adjacency_map, + scaled_matrix=scaled_matrix, + ) + if target_label is None: + continue + for component_index in disconnected_component: + working_labels[component_index] = target_label + merged_component_count += 1 + break + else: + raise DataDrivenSubdivisionError( + "نمی‌توان قید اتصال فضایی خوشه‌ها را تضمین کرد؛ بعضی سلول‌ها برای تشکیل بلوکِ دارای مرز مشترک قابل انتساب نبودند." + ) + + raise DataDrivenSubdivisionError( + "اعمال قید اتصال فضایی خوشه‌ها در تعداد تکرار مجاز همگرا نشد." + ) + + def build_cluster_summaries( *, observations: list[AnalysisGridObservation], @@ -392,12 +505,14 @@ def build_cluster_summaries( int(label), { "cluster_label": int(label), + "observations": [], "cell_codes": [], "centroid_lat_sum": 0.0, "centroid_lon_sum": 0.0, "cell_count": 0, }, ) + cluster["observations"].append(observation) cluster["cell_codes"].append(observation.cell.cell_code) cluster["centroid_lat_sum"] += float(observation.cell.centroid_lat) cluster["centroid_lon_sum"] += float(observation.cell.centroid_lon) @@ -407,6 +522,9 @@ def build_cluster_summaries( for cluster_label in sorted(clusters): cluster = clusters[cluster_label] cell_count = cluster["cell_count"] or 1 + center_payload = _select_cluster_center_observation( + cluster_observations=cluster["observations"], + ) summaries.append( { "cluster_label": cluster_label, @@ -414,11 +532,740 @@ def build_cluster_summaries( "centroid_lat": round(cluster["centroid_lat_sum"] / cell_count, 6), "centroid_lon": round(cluster["centroid_lon_sum"] / cell_count, 6), "cell_codes": cluster["cell_codes"], + "center_cell_code": center_payload["cell_code"], + "center_cell_lat": center_payload["centroid_lat"], + "center_cell_lon": center_payload["centroid_lon"], + "center_radius": center_payload["radius"], + "center_mean_distance": center_payload["mean_distance"], } ) return summaries +def _select_cluster_center_observation( + *, + cluster_observations: list[AnalysisGridObservation], +) -> dict[str, Any]: + if not cluster_observations: + return { + "cell_code": "", + "centroid_lat": None, + "centroid_lon": None, + "radius": 0.0, + "mean_distance": 0.0, + } + + candidate_payloads: list[dict[str, Any]] = [] + for candidate in cluster_observations: + candidate_lat = float(candidate.cell.centroid_lat) + candidate_lon = float(candidate.cell.centroid_lon) + distances = [ + _euclidean_distance( + [candidate_lon, candidate_lat], + [float(member.cell.centroid_lon), float(member.cell.centroid_lat)], + ) + for member in cluster_observations + ] + radius = max(distances) if distances else 0.0 + mean_distance = sum(distances) / len(distances) if distances else 0.0 + candidate_payloads.append( + { + "cell_code": candidate.cell.cell_code, + "centroid_lat": round(candidate_lat, 6), + "centroid_lon": round(candidate_lon, 6), + "radius": round(radius, 8), + "mean_distance": round(mean_distance, 8), + } + ) + + return min( + candidate_payloads, + key=lambda payload: ( + float(payload["radius"]), + float(payload["mean_distance"]), + str(payload["cell_code"]), + ), + ) + + +def build_subdivision_option_payloads( + *, + dataset: ClusteringDataset, + max_k: int, + random_state: int, +) -> list[SubdivisionOptionPayload]: + sample_count = len(dataset.observations) + if sample_count == 0: + return [] + max_allowed_k = min(max_k, sample_count) + option_payloads: list[SubdivisionOptionPayload] = [] + for requested_k in range(1, max_allowed_k + 1): + labels = run_kmeans_labels( + scaled_matrix=dataset.scaled_matrix, + cluster_count=requested_k, + random_state=random_state, + ) + labels, spatial_constraint_metadata = enforce_spatial_contiguity( + observations=dataset.observations, + labels=labels, + scaled_matrix=dataset.scaled_matrix, + ) + effective_cluster_count = len(set(labels)) + cluster_summaries = build_cluster_summaries( + observations=dataset.observations, + labels=labels, + ) + cluster_block_rows = build_cluster_block_rows( + observations=dataset.observations, + labels=labels, + cluster_summaries=cluster_summaries, + ) + assignment_rows = [ + { + "cell": observation.cell, + "cluster_label": int(labels[index]), + "raw_feature_values": dataset.raw_feature_maps[index], + "scaled_feature_values": { + feature_name: round(dataset.scaled_matrix[index][feature_index], 6) + for feature_index, feature_name in enumerate(dataset.selected_features) + }, + } + for index, observation in enumerate(dataset.observations) + ] + option_payloads.append( + SubdivisionOptionPayload( + requested_k=requested_k, + effective_cluster_count=effective_cluster_count, + labels=labels, + cluster_summaries=cluster_summaries, + spatial_constraint_metadata=spatial_constraint_metadata, + assignment_rows=assignment_rows, + cluster_block_rows=cluster_block_rows, + ) + ) + return option_payloads + + +def build_cluster_block_rows( + *, + observations: list[AnalysisGridObservation], + labels: list[int], + cluster_summaries: list[dict[str, Any]], +) -> list[dict[str, Any]]: + observations_by_label: dict[int, list[AnalysisGridObservation]] = {} + for observation, label in zip(observations, labels): + observations_by_label.setdefault(int(label), []).append(observation) + + cluster_block_rows: list[dict[str, Any]] = [] + for cluster_summary in cluster_summaries: + cluster_label = int(cluster_summary["cluster_label"]) + cluster_observations = observations_by_label.get(cluster_label, []) + cluster_geometry = _build_cluster_geometry(cluster_observations) + cluster_metadata = { + "cell_geometry_type": cluster_geometry.get("type"), + "source": "analysis_grid_cells", + } + cluster_block_rows.append( + { + "cluster_label": cluster_label, + "sub_block_code": f"cluster-{cluster_label}", + "centroid_lat": Decimal(str(cluster_summary["centroid_lat"])), + "centroid_lon": Decimal(str(cluster_summary["centroid_lon"])), + "center_cell_code": str(cluster_summary.get("center_cell_code") or ""), + "center_cell_lat": _to_decimal_or_none(cluster_summary.get("center_cell_lat")), + "center_cell_lon": _to_decimal_or_none(cluster_summary.get("center_cell_lon")), + "geometry": cluster_geometry, + "cell_count": int(cluster_summary["cell_count"]), + "cell_codes": list(cluster_summary["cell_codes"]), + "metadata": { + **cluster_metadata, + "center_selection": { + "strategy": "coordinate_1_center", + "center_cell_code": cluster_summary.get("center_cell_code") or "", + "center_radius": cluster_summary.get("center_radius"), + "center_mean_distance": cluster_summary.get("center_mean_distance"), + }, + }, + } + ) + return cluster_block_rows + + +def persist_subdivision_options( + *, + result: RemoteSensingSubdivisionResult, + location: SoilLocation, + block_subdivision: BlockSubdivision | None, + option_payloads: list[SubdivisionOptionPayload], + recommended_requested_k: int, + active_requested_k: int, + chunk_size_sqm: int, +) -> dict[int, RemoteSensingSubdivisionOption]: + option_objects: dict[int, RemoteSensingSubdivisionOption] = {} + for option_payload in option_payloads: + option = RemoteSensingSubdivisionOption.objects.create( + result=result, + requested_k=option_payload.requested_k, + effective_cluster_count=option_payload.effective_cluster_count, + is_active=option_payload.requested_k == active_requested_k, + is_recommended=option_payload.requested_k == recommended_requested_k, + selection_source="system", + metadata={ + "requested_k": option_payload.requested_k, + "effective_cluster_count": option_payload.effective_cluster_count, + "spatial_constraint": option_payload.spatial_constraint_metadata, + "cluster_summaries": option_payload.cluster_summaries, + }, + ) + RemoteSensingSubdivisionOptionAssignment.objects.bulk_create( + [ + RemoteSensingSubdivisionOptionAssignment( + option=option, + cell=assignment_row["cell"], + cluster_label=assignment_row["cluster_label"], + raw_feature_values=assignment_row["raw_feature_values"], + scaled_feature_values=assignment_row["scaled_feature_values"], + ) + for assignment_row in option_payload.assignment_rows + ] + ) + option_block_objects = [] + for block_row in option_payload.cluster_block_rows: + option_block_objects.append( + RemoteSensingSubdivisionOptionBlock( + option=option, + cluster_label=block_row["cluster_label"], + sub_block_code=block_row["sub_block_code"], + chunk_size_sqm=chunk_size_sqm, + centroid_lat=block_row["centroid_lat"], + centroid_lon=block_row["centroid_lon"], + center_cell_code=block_row["center_cell_code"], + center_cell_lat=block_row["center_cell_lat"], + center_cell_lon=block_row["center_cell_lon"], + geometry=block_row["geometry"], + cell_count=block_row["cell_count"], + cell_codes=block_row["cell_codes"], + metadata=block_row["metadata"], + ) + ) + RemoteSensingSubdivisionOptionBlock.objects.bulk_create(option_block_objects) + option_objects[option.requested_k] = option + return option_objects + + +def persist_subdivision_option_artifacts( + *, + result: RemoteSensingSubdivisionResult, + option_payloads: list[SubdivisionOptionPayload], + option_objects: dict[int, RemoteSensingSubdivisionOption], + observations: list[AnalysisGridObservation], + selected_features: list[str], + scaled_matrix: list[list[float]], + inertia_curve: list[dict[str, float]], +) -> None: + for option_payload in option_payloads: + option = option_objects.get(option_payload.requested_k) + if option is None: + continue + diagnostic_artifacts = _persist_remote_sensing_diagnostic_artifacts( + result=result, + observations=observations, + labels=option_payload.labels, + cluster_summaries=option_payload.cluster_summaries, + selected_features=selected_features, + scaled_matrix=scaled_matrix, + inertia_curve=inertia_curve, + requested_k=option_payload.requested_k, + effective_cluster_count=option_payload.effective_cluster_count, + ) + if not diagnostic_artifacts: + continue + metadata = dict(option.metadata or {}) + metadata["diagnostic_artifacts"] = diagnostic_artifacts + option.metadata = metadata + option.save(update_fields=["metadata", "updated_at"]) + + +def activate_subdivision_option( + *, + option: RemoteSensingSubdivisionOption, + selection_source: str, + recommended_requested_k: int | None = None, +) -> RemoteSensingSubdivisionResult: + result = option.result + requested_k = int(option.requested_k) + if recommended_requested_k is None: + recommended_requested_k = ( + result.options.filter(is_recommended=True) + .values_list("requested_k", flat=True) + .first() + ) + result.options.exclude(pk=option.pk).update(is_active=False) + option.is_active = True + option.selection_source = selection_source + option.save(update_fields=["is_active", "selection_source", "updated_at"]) + + assignments = list( + option.assignments.select_related("cell").order_by("cell__cell_code") + ) + result.assignments.all().delete() + RemoteSensingClusterAssignment.objects.bulk_create( + [ + RemoteSensingClusterAssignment( + result=result, + cell=assignment.cell, + cluster_label=assignment.cluster_label, + raw_feature_values=assignment.raw_feature_values, + scaled_feature_values=assignment.scaled_feature_values, + ) + for assignment in assignments + ] + ) + + result.cluster_blocks.all().delete() + cluster_block_objects = [] + cluster_summaries = [] + for option_block in option.cluster_blocks.order_by("cluster_label", "id"): + cluster_block = RemoteSensingClusterBlock.objects.create( + result=result, + soil_location=result.soil_location, + block_subdivision=result.block_subdivision, + block_code=result.block_code, + sub_block_code=option_block.sub_block_code, + cluster_label=option_block.cluster_label, + chunk_size_sqm=option_block.chunk_size_sqm, + centroid_lat=option_block.centroid_lat, + centroid_lon=option_block.centroid_lon, + center_cell_code=option_block.center_cell_code, + center_cell_lat=option_block.center_cell_lat, + center_cell_lon=option_block.center_cell_lon, + geometry=option_block.geometry, + cell_count=option_block.cell_count, + cell_codes=option_block.cell_codes, + metadata=option_block.metadata, + ) + cluster_block_objects.append(cluster_block) + cluster_summaries.append( + { + "cluster_uuid": str(cluster_block.uuid), + "cluster_label": option_block.cluster_label, + "sub_block_code": option_block.sub_block_code, + "centroid_lat": float(option_block.centroid_lat), + "centroid_lon": float(option_block.centroid_lon), + "center_cell_code": option_block.center_cell_code, + "center_cell_lat": float(option_block.center_cell_lat) if option_block.center_cell_lat is not None else None, + "center_cell_lon": float(option_block.center_cell_lon) if option_block.center_cell_lon is not None else None, + "center_radius": (option_block.metadata or {}).get("center_selection", {}).get("center_radius"), + "center_mean_distance": (option_block.metadata or {}).get("center_selection", {}).get("center_mean_distance"), + "cell_count": option_block.cell_count, + "cell_codes": list(option_block.cell_codes or []), + "geometry": option_block.geometry, + "metadata": option_block.metadata, + } + ) + + metadata = dict(result.metadata or {}) + kmeans_params = dict(metadata.get("kmeans_params") or {}) + kmeans_params["active_requested_k"] = requested_k + kmeans_params["effective_k"] = option.effective_cluster_count + if recommended_requested_k is not None: + kmeans_params["recommended_k"] = recommended_requested_k + metadata["kmeans_params"] = kmeans_params + metadata["active_requested_k"] = requested_k + metadata["recommended_requested_k"] = recommended_requested_k + metadata["cluster_summaries"] = cluster_summaries + metadata["active_option"] = { + "requested_k": requested_k, + "effective_cluster_count": option.effective_cluster_count, + "selection_source": selection_source, + } + metadata["available_k_options"] = [ + { + "requested_k": subdivision_option.requested_k, + "effective_cluster_count": subdivision_option.effective_cluster_count, + "is_active": subdivision_option.pk == option.pk, + "is_recommended": subdivision_option.is_recommended, + "selection_source": option.selection_source if subdivision_option.pk == option.pk else subdivision_option.selection_source, + "diagnostic_artifacts": (subdivision_option.metadata or {}).get("diagnostic_artifacts", {}), + } + for subdivision_option in result.options.order_by("requested_k") + ] + result.cluster_count = option.effective_cluster_count + result.metadata = metadata + result.save(update_fields=["cluster_count", "metadata", "updated_at"]) + if result.block_subdivision is not None: + sync_block_subdivision_with_result( + block_subdivision=result.block_subdivision, + result=result, + observations=assignments, + cluster_summaries=cluster_summaries, + ) + sync_location_block_layout_with_result( + location=result.soil_location, + result=result, + cluster_summaries=cluster_summaries, + ) + return result + + +def sync_cluster_blocks_with_result( + *, + result: RemoteSensingSubdivisionResult, + location: SoilLocation, + block_subdivision: BlockSubdivision | None, + observations: list[AnalysisGridObservation], + labels: list[int], + cluster_summaries: list[dict[str, Any]], +) -> list[RemoteSensingClusterBlock]: + observations_by_label: dict[int, list[AnalysisGridObservation]] = {} + for observation, label in zip(observations, labels): + observations_by_label.setdefault(int(label), []).append(observation) + + existing_blocks = { + cluster_block.cluster_label: cluster_block + for cluster_block in result.cluster_blocks.all() + } + active_labels: set[int] = set() + synced_blocks: list[RemoteSensingClusterBlock] = [] + for cluster_summary in cluster_summaries: + cluster_label = int(cluster_summary["cluster_label"]) + active_labels.add(cluster_label) + cluster_observations = observations_by_label.get(cluster_label, []) + cluster_geometry = _build_cluster_geometry(cluster_observations) + cluster_metadata = { + "cell_geometry_type": cluster_geometry.get("type"), + "source": "analysis_grid_cells", + } + cluster_block = existing_blocks.get(cluster_label) + defaults = { + "soil_location": location, + "block_subdivision": block_subdivision, + "block_code": result.block_code, + "sub_block_code": f"cluster-{cluster_label}", + "chunk_size_sqm": result.chunk_size_sqm, + "centroid_lat": Decimal(str(cluster_summary["centroid_lat"])), + "centroid_lon": Decimal(str(cluster_summary["centroid_lon"])), + "center_cell_code": str(cluster_summary.get("center_cell_code") or ""), + "center_cell_lat": _to_decimal_or_none(cluster_summary.get("center_cell_lat")), + "center_cell_lon": _to_decimal_or_none(cluster_summary.get("center_cell_lon")), + "geometry": cluster_geometry, + "cell_count": int(cluster_summary["cell_count"]), + "cell_codes": list(cluster_summary["cell_codes"]), + "metadata": { + **cluster_metadata, + "center_selection": { + "strategy": "coordinate_1_center", + "center_cell_code": cluster_summary.get("center_cell_code") or "", + "center_radius": cluster_summary.get("center_radius"), + "center_mean_distance": cluster_summary.get("center_mean_distance"), + }, + }, + } + if cluster_block is None: + cluster_block = RemoteSensingClusterBlock.objects.create( + result=result, + cluster_label=cluster_label, + **defaults, + ) + else: + for field_name, value in defaults.items(): + setattr(cluster_block, field_name, value) + cluster_block.save( + update_fields=[ + "soil_location", + "block_subdivision", + "block_code", + "sub_block_code", + "chunk_size_sqm", + "centroid_lat", + "centroid_lon", + "center_cell_code", + "center_cell_lat", + "center_cell_lon", + "geometry", + "cell_count", + "cell_codes", + "metadata", + "updated_at", + ] + ) + cluster_summary["cluster_uuid"] = str(cluster_block.uuid) + cluster_summary["geometry"] = cluster_block.geometry + cluster_summary["metadata"] = cluster_block.metadata + synced_blocks.append(cluster_block) + + stale_labels = set(existing_blocks) - active_labels + if stale_labels: + result.cluster_blocks.filter(cluster_label__in=stale_labels).delete() + return synced_blocks + + +def _build_cluster_geometry( + observations: list[AnalysisGridObservation], +) -> dict[str, Any]: + rings = _build_cluster_boundary_rings(observations) + if not rings: + return {} + if len(rings) == 1: + return {"type": "Polygon", "coordinates": [rings[0]]} + + outer_ring_indexes = [] + hole_mapping: dict[int, list[list[list[float]]]] = {} + for ring_index, ring in enumerate(rings): + sample_point = ring[0] + parent_indexes = [ + candidate_index + for candidate_index, candidate_ring in enumerate(rings) + if candidate_index != ring_index and point_in_polygon( + (sample_point[0], sample_point[1]), + [(point[0], point[1]) for point in candidate_ring[:-1]], + ) + ] + if not parent_indexes: + outer_ring_indexes.append(ring_index) + continue + parent_index = min( + parent_indexes, + key=lambda candidate_index: abs(_signed_ring_area(rings[candidate_index])), + ) + hole_mapping.setdefault(parent_index, []).append(ring) + + if len(outer_ring_indexes) == 1: + outer_index = outer_ring_indexes[0] + return { + "type": "Polygon", + "coordinates": [rings[outer_index], *hole_mapping.get(outer_index, [])], + } + + return { + "type": "MultiPolygon", + "coordinates": [ + [rings[outer_index], *hole_mapping.get(outer_index, [])] + for outer_index in outer_ring_indexes + ], + } + + +def _build_shared_border_adjacency( + observations: list[AnalysisGridObservation], +) -> dict[int, set[int]]: + adjacency_map: dict[int, set[int]] = {index: set() for index in range(len(observations))} + shared_edge_map: dict[tuple[tuple[float, float], tuple[float, float]], list[int]] = {} + for index, observation in enumerate(observations): + for edge_key in _extract_cell_shared_edge_keys(observation): + shared_edge_map.setdefault(edge_key, []).append(index) + + for cell_indexes in shared_edge_map.values(): + if len(cell_indexes) != 2: + continue + left_index, right_index = cell_indexes + adjacency_map[left_index].add(right_index) + adjacency_map[right_index].add(left_index) + return adjacency_map + + +def _extract_cell_shared_edge_keys( + observation: AnalysisGridObservation, +) -> set[tuple[tuple[float, float], tuple[float, float]]]: + geometry = dict(getattr(observation.cell, "geometry", {}) or {}) + coordinates = geometry.get("coordinates") or [] + polygons = [] + if geometry.get("type") == "Polygon" and coordinates: + polygons = [coordinates] + elif geometry.get("type") == "MultiPolygon" and coordinates: + polygons = coordinates + + edge_keys: set[tuple[tuple[float, float], tuple[float, float]]] = set() + for polygon in polygons: + outer_ring = polygon[0] if polygon else [] + normalized_ring = [ + (float(point[0]), float(point[1])) + for point in outer_ring + if len(point) >= 2 + ] + if len(normalized_ring) < 4: + continue + if normalized_ring[0] != normalized_ring[-1]: + normalized_ring.append(normalized_ring[0]) + for start_point, end_point in zip(normalized_ring, normalized_ring[1:]): + if start_point == end_point: + continue + edge_keys.add(tuple(sorted((start_point, end_point)))) + return edge_keys + + +def _find_disconnected_label_components( + *, + labels: list[int], + adjacency_map: dict[int, set[int]], +) -> list[list[int]]: + components_to_merge: list[list[int]] = [] + for cluster_label in sorted(set(labels)): + label_indexes = [index for index, label in enumerate(labels) if int(label) == cluster_label] + if len(label_indexes) <= 1: + continue + label_index_set = set(label_indexes) + visited: set[int] = set() + connected_components: list[list[int]] = [] + for start_index in label_indexes: + if start_index in visited: + continue + component = [] + queue = [start_index] + visited.add(start_index) + while queue: + current_index = queue.pop() + component.append(current_index) + for neighbor_index in adjacency_map.get(current_index, set()): + if neighbor_index in visited or neighbor_index not in label_index_set: + continue + visited.add(neighbor_index) + queue.append(neighbor_index) + connected_components.append(sorted(component)) + if len(connected_components) <= 1: + continue + connected_components.sort(key=lambda component: (-len(component), component[0])) + components_to_merge.extend(connected_components[1:]) + return components_to_merge + + +def _choose_neighbor_label_for_component( + *, + component_indexes: list[int], + labels: list[int], + adjacency_map: dict[int, set[int]], + scaled_matrix: list[list[float]], +) -> int | None: + component_centroid = _mean_vector([scaled_matrix[index] for index in component_indexes]) + candidate_scores: dict[int, float] = {} + for component_index in component_indexes: + for neighbor_index in adjacency_map.get(component_index, set()): + neighbor_label = int(labels[neighbor_index]) + if neighbor_label == int(labels[component_index]): + continue + candidate_indexes = [ + index + for index, label in enumerate(labels) + if int(label) == neighbor_label + ] + if not candidate_indexes: + continue + candidate_centroid = _mean_vector([scaled_matrix[index] for index in candidate_indexes]) + distance = _euclidean_distance(component_centroid, candidate_centroid) + best_distance = candidate_scores.get(neighbor_label) + if best_distance is None or distance < best_distance: + candidate_scores[neighbor_label] = distance + if not candidate_scores: + return None + return min(candidate_scores.items(), key=lambda item: (item[1], item[0]))[0] + + +def _normalize_cluster_labels(labels: list[int]) -> list[int]: + label_mapping: dict[int, int] = {} + normalized_labels: list[int] = [] + next_label = 0 + for label in labels: + label = int(label) + if label not in label_mapping: + label_mapping[label] = next_label + next_label += 1 + normalized_labels.append(label_mapping[label]) + return normalized_labels + + +def _mean_vector(vectors: list[list[float]]) -> list[float]: + if not vectors: + return [] + dimensions = len(vectors[0]) + return [ + sum(vector[dimension_index] for vector in vectors) / len(vectors) + for dimension_index in range(dimensions) + ] + + +def _euclidean_distance(left: list[float], right: list[float]) -> float: + return math.sqrt( + sum((float(left[index]) - float(right[index])) ** 2 for index in range(len(left))) + ) + + +def _build_cluster_boundary_rings( + observations: list[AnalysisGridObservation], +) -> list[list[list[float]]]: + directed_edges: dict[tuple[tuple[float, float], tuple[float, float]], int] = {} + undirected_counts: dict[tuple[tuple[float, float], tuple[float, float]], int] = {} + point_lookup: dict[tuple[float, float], list[tuple[float, float]]] = {} + + for observation in observations: + geometry = dict(getattr(observation.cell, "geometry", {}) or {}) + coordinates = geometry.get("coordinates") or [] + polygons = [] + if geometry.get("type") == "Polygon" and coordinates: + polygons = [coordinates] + elif geometry.get("type") == "MultiPolygon" and coordinates: + polygons = coordinates + for polygon in polygons: + outer_ring = polygon[0] if polygon else [] + normalized_ring = [ + (float(point[0]), float(point[1])) + for point in outer_ring + if len(point) >= 2 + ] + if len(normalized_ring) < 4: + continue + if normalized_ring[0] != normalized_ring[-1]: + normalized_ring.append(normalized_ring[0]) + for start_point, end_point in zip(normalized_ring, normalized_ring[1:]): + if start_point == end_point: + continue + directed_edges[(start_point, end_point)] = directed_edges.get((start_point, end_point), 0) + 1 + undirected_key = tuple(sorted((start_point, end_point))) + undirected_counts[undirected_key] = undirected_counts.get(undirected_key, 0) + 1 + + boundary_edges = [ + (start_point, end_point) + for (start_point, end_point), _count in directed_edges.items() + if undirected_counts.get(tuple(sorted((start_point, end_point))), 0) == 1 + ] + if not boundary_edges: + return [] + + for start_point, end_point in boundary_edges: + point_lookup.setdefault(start_point, []).append(end_point) + + rings: list[list[list[float]]] = [] + while point_lookup: + start_point = next(iter(point_lookup)) + current_point = start_point + ring = [start_point] + visited_guard = 0 + while visited_guard <= len(boundary_edges) + 1: + next_points = point_lookup.get(current_point) or [] + if not next_points: + break + next_point = next_points.pop(0) + if not next_points: + point_lookup.pop(current_point, None) + current_point = next_point + ring.append(current_point) + if current_point == start_point: + break + visited_guard += 1 + if len(ring) >= 4 and ring[0] == ring[-1]: + signed_area = _signed_ring_area(ring) + if signed_area < 0: + ring = [ring[0], *list(reversed(ring[1:-1])), ring[0]] + rings.append([[point[0], point[1]] for point in ring]) + return rings + + +def _signed_ring_area(ring: list[list[float]] | list[tuple[float, float]]) -> float: + area = 0.0 + for current_point, next_point in zip(ring, ring[1:]): + area += (float(current_point[0]) * float(next_point[1])) - (float(next_point[0]) * float(current_point[1])) + return area / 2.0 + + def sync_location_block_layout_with_result( *, location: SoilLocation, @@ -446,11 +1293,17 @@ def sync_location_block_layout_with_result( target_block["needs_subdivision"] = result.cluster_count > 1 target_block["sub_blocks"] = [ { + "cluster_uuid": cluster.get("cluster_uuid"), "sub_block_code": f"cluster-{cluster['cluster_label']}", "cluster_label": cluster["cluster_label"], "centroid_lat": cluster["centroid_lat"], "centroid_lon": cluster["centroid_lon"], + "center_cell_code": cluster.get("center_cell_code") or "", + "center_cell_lat": cluster.get("center_cell_lat"), + "center_cell_lon": cluster.get("center_cell_lon"), "cell_count": cluster["cell_count"], + "geometry": cluster.get("geometry") or {}, + "metadata": cluster.get("metadata") or {}, } for cluster in cluster_summaries ] @@ -501,12 +1354,18 @@ def sync_block_subdivision_with_result( ] block_subdivision.centroid_points = [ { + "cluster_uuid": cluster.get("cluster_uuid"), "sub_block_code": f"cluster-{cluster['cluster_label']}", "cluster_label": cluster["cluster_label"], "centroid_lat": cluster["centroid_lat"], "centroid_lon": cluster["centroid_lon"], + "center_cell_code": cluster.get("center_cell_code") or "", + "center_cell_lat": cluster.get("center_cell_lat"), + "center_cell_lon": cluster.get("center_cell_lon"), "cell_count": cluster["cell_count"], "cell_codes": cluster["cell_codes"], + "geometry": cluster.get("geometry") or {}, + "metadata": cluster.get("metadata") or {}, } for cluster in cluster_summaries ] @@ -562,6 +1421,15 @@ def _coerce_float(value: Any) -> float | None: return None +def _to_decimal_or_none(value: Any) -> Decimal | None: + if value is None: + return None + try: + return Decimal(str(value)) + except (ArithmeticError, ValueError): + return None + + def _count_non_null_features(observations: list[AnalysisGridObservation]) -> dict[str, int]: counts = {feature_name: 0 for feature_name in DEFAULT_CLUSTER_FEATURES} for observation in observations: @@ -580,9 +1448,15 @@ def _persist_remote_sensing_diagnostic_artifacts( selected_features: list[str], scaled_matrix: list[list[float]], inertia_curve: list[dict[str, float]], + requested_k: int | None = None, + effective_cluster_count: int | None = None, ) -> dict[str, Any]: try: - artifact_dir = _build_remote_sensing_diagnostic_dir(result=result) + artifact_dir = _build_remote_sensing_diagnostic_dir( + result=result, + requested_k=requested_k, + effective_cluster_count=effective_cluster_count, + ) artifact_dir.mkdir(parents=True, exist_ok=True) specs = [ @@ -600,6 +1474,7 @@ def _persist_remote_sensing_diagnostic_artifacts( _render_cluster_map_plot( observations=observations, labels=labels, + cluster_summaries=cluster_summaries, block_code=result.block_code or "farm", ), "cluster-map", @@ -607,6 +1482,7 @@ def _persist_remote_sensing_diagnostic_artifacts( ( "cluster_sizes", _render_cluster_size_plot( + observations=observations, cluster_summaries=cluster_summaries, block_code=result.block_code or "farm", ), @@ -615,26 +1491,45 @@ def _persist_remote_sensing_diagnostic_artifacts( ( "feature_pairs", _render_feature_pair_plot( + observations=observations, selected_features=selected_features, scaled_matrix=scaled_matrix, labels=labels, + cluster_summaries=cluster_summaries, block_code=result.block_code or "farm", ), "feature-pairs", ), + ( + "feature_projection", + _render_feature_projection_plot( + observations=observations, + selected_features=selected_features, + scaled_matrix=scaled_matrix, + labels=labels, + cluster_summaries=cluster_summaries, + block_code=result.block_code or "farm", + ), + "feature-projection", + ), ] files: dict[str, str] = {} for artifact_key, content, suffix in specs: if content is None: continue - target_path = artifact_dir / f"{_build_remote_sensing_artifact_stem(result=result)}__{suffix}.png" + target_path = artifact_dir / ( + f"{_build_remote_sensing_artifact_stem(result=result, requested_k=requested_k, effective_cluster_count=effective_cluster_count)}" + f"__{suffix}.png" + ) _write_content_file(target_path=target_path, content=content) files[artifact_key] = _to_project_relative_path(target_path) return { "directory": _to_project_relative_path(artifact_dir), "files": files, + "requested_k": requested_k, + "effective_cluster_count": effective_cluster_count, } except (DataDrivenSubdivisionError, OSError) as exc: logger.warning( @@ -645,7 +1540,12 @@ def _persist_remote_sensing_diagnostic_artifacts( return {} -def _build_remote_sensing_diagnostic_dir(*, result: RemoteSensingSubdivisionResult) -> Path: +def _build_remote_sensing_diagnostic_dir( + *, + result: RemoteSensingSubdivisionResult, + requested_k: int | None = None, + effective_cluster_count: int | None = None, +) -> Path: configured_dir = str( os.environ.get("REMOTE_SENSING_DIAGNOSTIC_DIR", DEFAULT_REMOTE_SENSING_DIAGNOSTIC_DIR) ).strip() @@ -654,15 +1554,28 @@ def _build_remote_sensing_diagnostic_dir(*, result: RemoteSensingSubdivisionResu if not target_dir.is_absolute(): target_dir = base_dir / target_dir block_component = _sanitize_path_component(result.block_code or "farm") - return target_dir / f"location-{result.soil_location_id}" / f"run-{result.run_id}-{block_component}" + diagnostic_dir = target_dir / f"location-{result.soil_location_id}" / f"run-{result.run_id}-{block_component}" + if requested_k is not None: + effective_component = effective_cluster_count if effective_cluster_count is not None else requested_k + diagnostic_dir = diagnostic_dir / f"k-{requested_k}-effective-{effective_component}" + return diagnostic_dir -def _build_remote_sensing_artifact_stem(*, result: RemoteSensingSubdivisionResult) -> str: - return ( +def _build_remote_sensing_artifact_stem( + *, + result: RemoteSensingSubdivisionResult, + requested_k: int | None = None, + effective_cluster_count: int | None = None, +) -> str: + stem = ( f"location-{result.soil_location_id}" f"__run-{result.run_id}" f"__{_sanitize_path_component(result.block_code or 'farm')}" ) + if requested_k is not None: + effective_component = effective_cluster_count if effective_cluster_count is not None else requested_k + stem = f"{stem}__k-{requested_k}__effective-{effective_component}" + return stem def _write_content_file(*, target_path: Path, content: ContentFile) -> None: @@ -692,6 +1605,7 @@ def _render_cluster_map_plot( *, observations: list[AnalysisGridObservation], labels: list[int], + cluster_summaries: list[dict[str, Any]], block_code: str, ) -> ContentFile | None: if not observations: @@ -699,13 +1613,22 @@ def _render_cluster_map_plot( plt = _import_matplotlib_pyplot() unique_labels = sorted(set(int(label) for label in labels)) colors = plt.cm.get_cmap("tab10", max(len(unique_labels), 1)) + center_indexes_by_label = _build_center_indexes_by_label( + observations=observations, + labels=labels, + cluster_summaries=cluster_summaries, + ) fig, ax = plt.subplots(figsize=(8, 6)) buffer = BytesIO() try: for color_index, cluster_label in enumerate(unique_labels): cluster_points = [ - (float(observation.cell.centroid_lon), float(observation.cell.centroid_lat)) - for observation, label in zip(observations, labels) + ( + float(observation.cell.centroid_lon), + float(observation.cell.centroid_lat), + _build_observation_label(observation=observation, index=index), + ) + for index, (observation, label) in enumerate(zip(observations, labels)) if int(label) == cluster_label ] if not cluster_points: @@ -722,6 +1645,25 @@ def _render_cluster_map_plot( linewidths=0.8, label=f"Cluster {cluster_label}", ) + _annotate_plot_points( + axis=ax, + x_values=xs, + y_values=ys, + point_labels=[point[2] for point in cluster_points], + ) + center_index = center_indexes_by_label.get(cluster_label) + if center_index is not None: + _plot_cluster_center_marker( + axis=ax, + x_value=float(observations[center_index].cell.centroid_lon), + y_value=float(observations[center_index].cell.centroid_lat), + point_label=_build_center_label( + observations=observations, + cluster_summaries=cluster_summaries, + cluster_label=cluster_label, + ), + color=colors(color_index), + ) ax.set_title(f"KMeans Spatial Cluster Map - {block_code}") ax.set_xlabel("Longitude") ax.set_ylabel("Latitude") @@ -739,6 +1681,7 @@ def _render_cluster_map_plot( def _render_cluster_size_plot( *, + observations: list[AnalysisGridObservation], cluster_summaries: list[dict[str, Any]], block_code: str, ) -> ContentFile | None: @@ -751,6 +1694,10 @@ def _render_cluster_size_plot( buffer = BytesIO() try: bars = ax.bar(labels, counts, color="#2f6fed", alpha=0.85) + point_numbers_by_cell_code = { + str(observation.cell.cell_code): index + 1 + for index, observation in enumerate(observations) + } for bar, count in zip(bars, counts): ax.text( bar.get_x() + bar.get_width() / 2.0, @@ -760,6 +1707,21 @@ def _render_cluster_size_plot( va="bottom", fontsize=9, ) + for bar, cluster_summary in zip(bars, cluster_summaries): + center_cell_code = str(cluster_summary.get("center_cell_code") or "").strip() + if center_cell_code: + center_point_number = point_numbers_by_cell_code.get(center_cell_code) + center_text = f"center: {center_point_number}" if center_point_number is not None else "center" + ax.text( + bar.get_x() + bar.get_width() / 2.0, + bar.get_height() / 2.0 if bar.get_height() else 0.05, + center_text, + ha="center", + va="center", + fontsize=8, + color="#16325c", + rotation=90, + ) ax.set_title(f"Cluster Sizes - {block_code}") ax.set_xlabel("Cluster") ax.set_ylabel("Cell Count") @@ -775,9 +1737,11 @@ def _render_cluster_size_plot( def _render_feature_pair_plot( *, + observations: list[AnalysisGridObservation], selected_features: list[str], scaled_matrix: list[list[float]], labels: list[int], + cluster_summaries: list[dict[str, Any]], block_code: str, ) -> ContentFile | None: if not scaled_matrix or not selected_features: @@ -796,6 +1760,15 @@ def _render_feature_pair_plot( axes_list = axes.flatten().tolist() if hasattr(axes, "flatten") else [axes] unique_labels = sorted(set(int(label) for label in labels)) colors = plt.cm.get_cmap("tab10", max(len(unique_labels), 1)) + observation_labels = [ + _build_observation_label(observation=observation, index=index) + for index, observation in enumerate(observations) + ] + center_indexes_by_label = _build_center_indexes_by_label( + observations=observations, + labels=labels, + cluster_summaries=cluster_summaries, + ) buffer = BytesIO() try: for axis, (left_index, right_index) in zip(axes_list, pair_indexes): @@ -804,8 +1777,8 @@ def _render_feature_pair_plot( ys = [row[0] for row in scaled_matrix] for color_index, cluster_label in enumerate(unique_labels): filtered = [ - (x_value, y_value) - for x_value, y_value, label in zip(xs, ys, labels) + (x_value, y_value, point_label) + for x_value, y_value, label, point_label in zip(xs, ys, labels, observation_labels) if int(label) == cluster_label ] axis.scatter( @@ -816,6 +1789,25 @@ def _render_feature_pair_plot( alpha=0.85, label=f"Cluster {cluster_label}", ) + _annotate_plot_points( + axis=axis, + x_values=[item[0] for item in filtered], + y_values=[item[1] for item in filtered], + point_labels=[item[2] for item in filtered], + ) + center_index = center_indexes_by_label.get(cluster_label) + if center_index is not None: + _plot_cluster_center_marker( + axis=axis, + x_value=float(center_index + 1), + y_value=float(scaled_matrix[center_index][0]), + point_label=_build_center_label( + observations=observations, + cluster_summaries=cluster_summaries, + cluster_label=cluster_label, + ), + color=colors(color_index), + ) axis.set_xlabel("Observation Index") axis.set_ylabel(f"{selected_features[0]} (scaled)") axis.set_title(f"{selected_features[0]} distribution") @@ -824,8 +1816,13 @@ def _render_feature_pair_plot( y_values = [row[right_index] for row in scaled_matrix] for color_index, cluster_label in enumerate(unique_labels): filtered = [ - (x_value, y_value) - for x_value, y_value, label in zip(x_values, y_values, labels) + (x_value, y_value, point_label) + for x_value, y_value, label, point_label in zip( + x_values, + y_values, + labels, + observation_labels, + ) if int(label) == cluster_label ] axis.scatter( @@ -836,6 +1833,25 @@ def _render_feature_pair_plot( alpha=0.85, label=f"Cluster {cluster_label}", ) + _annotate_plot_points( + axis=axis, + x_values=[item[0] for item in filtered], + y_values=[item[1] for item in filtered], + point_labels=[item[2] for item in filtered], + ) + center_index = center_indexes_by_label.get(cluster_label) + if center_index is not None: + _plot_cluster_center_marker( + axis=axis, + x_value=float(scaled_matrix[center_index][left_index]), + y_value=float(scaled_matrix[center_index][right_index]), + point_label=_build_center_label( + observations=observations, + cluster_summaries=cluster_summaries, + cluster_label=cluster_label, + ), + color=colors(color_index), + ) axis.set_xlabel(f"{selected_features[left_index]} (scaled)") axis.set_ylabel(f"{selected_features[right_index]} (scaled)") axis.set_title( @@ -858,6 +1874,234 @@ def _render_feature_pair_plot( plt.close(fig) +def _render_feature_projection_plot( + *, + observations: list[AnalysisGridObservation], + selected_features: list[str], + scaled_matrix: list[list[float]], + labels: list[int], + cluster_summaries: list[dict[str, Any]], + block_code: str, +) -> ContentFile | None: + if not scaled_matrix: + return None + plt = _import_matplotlib_pyplot() + projected_points, x_label, y_label = _project_all_features_to_2d( + scaled_matrix=scaled_matrix, + selected_features=selected_features, + ) + unique_labels = sorted(set(int(label) for label in labels)) + colors = plt.cm.get_cmap("tab10", max(len(unique_labels), 1)) + observation_labels = [ + _build_observation_label(observation=observation, index=index) + for index, observation in enumerate(observations) + ] + center_indexes_by_label = _build_center_indexes_by_label( + observations=observations, + labels=labels, + cluster_summaries=cluster_summaries, + ) + fig, ax = plt.subplots(figsize=(8, 6)) + buffer = BytesIO() + try: + for color_index, cluster_label in enumerate(unique_labels): + filtered = [ + (x_value, y_value, point_label) + for (x_value, y_value), label, point_label in zip(projected_points, labels, observation_labels) + if int(label) == cluster_label + ] + if not filtered: + continue + ax.scatter( + [item[0] for item in filtered], + [item[1] for item in filtered], + s=65, + color=colors(color_index), + alpha=0.9, + edgecolors="white", + linewidths=0.8, + label=f"Cluster {cluster_label}", + ) + _annotate_plot_points( + axis=ax, + x_values=[item[0] for item in filtered], + y_values=[item[1] for item in filtered], + point_labels=[item[2] for item in filtered], + ) + center_index = center_indexes_by_label.get(cluster_label) + if center_index is not None and center_index < len(projected_points): + _plot_cluster_center_marker( + axis=ax, + x_value=float(projected_points[center_index][0]), + y_value=float(projected_points[center_index][1]), + point_label=_build_center_label( + observations=observations, + cluster_summaries=cluster_summaries, + cluster_label=cluster_label, + ), + color=colors(color_index), + ) + ax.set_title(f"KMeans All-Feature Projection - {block_code}") + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.4) + if unique_labels: + ax.legend() + fig.tight_layout() + fig.savefig(buffer, format="png", dpi=150) + buffer.seek(0) + return ContentFile(buffer.getvalue()) + finally: + buffer.close() + plt.close(fig) + + +def _project_all_features_to_2d( + *, + scaled_matrix: list[list[float]], + selected_features: list[str], +) -> tuple[list[tuple[float, float]], str, str]: + if not scaled_matrix: + return [], "Projection Axis 1", "Projection Axis 2" + + matrix = [[float(value) for value in row] for row in scaled_matrix] + row_count = len(matrix) + column_count = len(matrix[0]) if matrix and matrix[0] else 0 + + if row_count >= 2 and column_count >= 2: + try: + from sklearn.decomposition import PCA + + pca = PCA(n_components=2) + projected_matrix = pca.fit_transform(matrix) + x_axis_label = "PC1" + y_axis_label = "PC2" + explained = list(getattr(pca, "explained_variance_ratio_", []) or []) + if len(explained) >= 2: + x_axis_label = f"PC1 ({explained[0] * 100:.1f}%)" + y_axis_label = f"PC2 ({explained[1] * 100:.1f}%)" + return ( + [(float(row[0]), float(row[1])) for row in projected_matrix.tolist()], + x_axis_label, + y_axis_label, + ) + except ImportError: + logger.warning( + "scikit-learn PCA is unavailable, falling back to the first scaled features for projection." + ) + except ValueError as exc: + logger.warning("Failed to calculate PCA projection for remote sensing diagnostics: %s", exc) + + x_values = [float(row[0]) if row else 0.0 for row in matrix] + if column_count >= 2: + y_values = [float(row[1]) for row in matrix] + else: + y_values = [0.0 for _ in matrix] + x_axis_label = f"{selected_features[0]} (scaled)" if selected_features else "Feature 1 (scaled)" + y_axis_label = ( + f"{selected_features[1]} (scaled)" + if len(selected_features) >= 2 + else "Projection Axis 2" + ) + return list(zip(x_values, y_values)), x_axis_label, y_axis_label + + +def _build_observation_label(*, observation: AnalysisGridObservation, index: int) -> str: + _ = observation + return str(index + 1) + + +def _annotate_plot_points( + *, + axis: Any, + x_values: list[float], + y_values: list[float], + point_labels: list[str], +) -> None: + for x_value, y_value, point_label in zip(x_values, y_values, point_labels): + axis.annotate( + point_label, + xy=(x_value, y_value), + xytext=(4, 4), + textcoords="offset points", + fontsize=7, + alpha=0.85, + ) + + +def _build_center_indexes_by_label( + *, + observations: list[AnalysisGridObservation], + labels: list[int], + cluster_summaries: list[dict[str, Any]], +) -> dict[int, int]: + cell_code_to_index = { + str(observation.cell.cell_code): index + for index, observation in enumerate(observations) + } + center_indexes_by_label: dict[int, int] = {} + for cluster_summary in cluster_summaries: + cluster_label = int(cluster_summary.get("cluster_label", -1)) + center_cell_code = str(cluster_summary.get("center_cell_code") or "").strip() + center_index = cell_code_to_index.get(center_cell_code) + if center_index is None: + continue + if center_index < len(labels) and int(labels[center_index]) == cluster_label: + center_indexes_by_label[cluster_label] = center_index + return center_indexes_by_label + + +def _build_center_label( + *, + observations: list[AnalysisGridObservation], + cluster_summaries: list[dict[str, Any]], + cluster_label: int, +) -> str: + point_numbers_by_cell_code = { + str(observation.cell.cell_code): index + 1 + for index, observation in enumerate(observations) + } + for cluster_summary in cluster_summaries: + if int(cluster_summary.get("cluster_label", -1)) == int(cluster_label): + center_cell_code = str(cluster_summary.get("center_cell_code") or "").strip() + if center_cell_code: + point_number = point_numbers_by_cell_code.get(center_cell_code) + if point_number is not None: + return f"K-center: {point_number}" + return "K-center" + break + return f"K-center C{cluster_label}" + + +def _plot_cluster_center_marker( + *, + axis: Any, + x_value: float, + y_value: float, + point_label: str, + color: Any, +) -> None: + axis.scatter( + [x_value], + [y_value], + s=220, + marker="*", + color=color, + edgecolors="black", + linewidths=1.2, + zorder=5, + ) + axis.annotate( + point_label, + xy=(x_value, y_value), + xytext=(7, -10), + textcoords="offset points", + fontsize=8, + fontweight="bold", + color="black", + ) + + def _import_matplotlib_pyplot(): try: import matplotlib diff --git a/location_data/migrations/0017_remotesensingclusterblock.py b/location_data/migrations/0017_remotesensingclusterblock.py new file mode 100644 index 0000000..f52ffca --- /dev/null +++ b/location_data/migrations/0017_remotesensingclusterblock.py @@ -0,0 +1,48 @@ +import uuid + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("location_data", "0016_remove_analysisgridobservation_lst_c"), + ] + + operations = [ + migrations.CreateModel( + name="RemoteSensingClusterBlock", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("uuid", models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True)), + ("block_code", models.CharField(blank=True, db_index=True, default="", help_text="شناسه بلوک والد که این زیر‌بلاک KMeans داخل آن ساخته شده است.", max_length=64)), + ("sub_block_code", models.CharField(db_index=True, help_text="شناسه زیر‌بلاک ساخته‌شده توسط KMeans مثل cluster-0.", max_length=64)), + ("cluster_label", models.PositiveIntegerField(db_index=True)), + ("chunk_size_sqm", models.PositiveIntegerField(default=900)), + ("centroid_lat", models.DecimalField(db_index=True, decimal_places=6, help_text="عرض جغرافیایی مرکز زیر‌بلاک.", max_digits=9)), + ("centroid_lon", models.DecimalField(db_index=True, decimal_places=6, help_text="طول جغرافیایی مرکز زیر‌بلاک.", max_digits=9)), + ("geometry", models.JSONField(blank=True, default=dict, help_text="هندسه GeoJSON زیر‌بلاک KMeans. فعلا از چندضلعی/چندچندضلعی سلول‌های عضو ساخته می‌شود.")), + ("cell_count", models.PositiveIntegerField(default=0)), + ("cell_codes", models.JSONField(blank=True, default=list)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("block_subdivision", models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, related_name="remote_sensing_cluster_blocks", to="location_data.blocksubdivision")), + ("result", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="cluster_blocks", to="location_data.remotesensingsubdivisionresult")), + ("soil_location", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="remote_sensing_cluster_blocks", to="location_data.soillocation")), + ], + options={ + "verbose_name": "remote sensing cluster block", + "verbose_name_plural": "remote sensing cluster blocks", + "ordering": ["result", "cluster_label", "id"], + }, + ), + migrations.AddConstraint( + model_name="remotesensingclusterblock", + constraint=models.UniqueConstraint(fields=("result", "cluster_label"), name="rs_cluster_block_unique_result_label"), + ), + migrations.AddIndex( + model_name="remotesensingclusterblock", + index=models.Index(fields=["soil_location", "block_code", "cluster_label"], name="rs_cluster_block_lookup_idx"), + ), + ] diff --git a/location_data/migrations/0018_remotesensingsubdivisionoption_and_more.py b/location_data/migrations/0018_remotesensingsubdivisionoption_and_more.py new file mode 100644 index 0000000..80ecb46 --- /dev/null +++ b/location_data/migrations/0018_remotesensingsubdivisionoption_and_more.py @@ -0,0 +1,92 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("location_data", "0017_remotesensingclusterblock"), + ] + + operations = [ + migrations.CreateModel( + name="RemoteSensingSubdivisionOption", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("requested_k", models.PositiveIntegerField(db_index=True)), + ("effective_cluster_count", models.PositiveIntegerField(default=0)), + ("is_active", models.BooleanField(db_index=True, default=False)), + ("is_recommended", models.BooleanField(db_index=True, default=False)), + ("selection_source", models.CharField(default="system", help_text="منشا انتخاب این گزینه؛ مثل system یا user.", max_length=32)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("result", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="options", to="location_data.remotesensingsubdivisionresult")), + ], + options={ + "verbose_name": "remote sensing subdivision option", + "verbose_name_plural": "remote sensing subdivision options", + "ordering": ["result", "requested_k", "id"], + }, + ), + migrations.CreateModel( + name="RemoteSensingSubdivisionOptionBlock", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("cluster_label", models.PositiveIntegerField(db_index=True)), + ("sub_block_code", models.CharField(db_index=True, max_length=64)), + ("chunk_size_sqm", models.PositiveIntegerField(default=900)), + ("centroid_lat", models.DecimalField(db_index=True, decimal_places=6, max_digits=9)), + ("centroid_lon", models.DecimalField(db_index=True, decimal_places=6, max_digits=9)), + ("geometry", models.JSONField(blank=True, default=dict)), + ("cell_count", models.PositiveIntegerField(default=0)), + ("cell_codes", models.JSONField(blank=True, default=list)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("option", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="cluster_blocks", to="location_data.remotesensingsubdivisionoption")), + ], + options={ + "verbose_name": "remote sensing subdivision option block", + "verbose_name_plural": "remote sensing subdivision option blocks", + "ordering": ["option", "cluster_label", "id"], + }, + ), + migrations.CreateModel( + name="RemoteSensingSubdivisionOptionAssignment", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("cluster_label", models.PositiveIntegerField(db_index=True)), + ("raw_feature_values", models.JSONField(blank=True, default=dict)), + ("scaled_feature_values", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("cell", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="subdivision_option_assignments", to="location_data.analysisgridcell")), + ("option", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="assignments", to="location_data.remotesensingsubdivisionoption")), + ], + options={ + "verbose_name": "remote sensing subdivision option assignment", + "verbose_name_plural": "remote sensing subdivision option assignments", + "ordering": ["option", "cluster_label", "cell__cell_code"], + }, + ), + migrations.AddConstraint( + model_name="remotesensingsubdivisionoption", + constraint=models.UniqueConstraint(fields=("result", "requested_k"), name="rs_subdiv_option_unique_result_requested_k"), + ), + migrations.AddIndex( + model_name="remotesensingsubdivisionoption", + index=models.Index(fields=["result", "is_active"], name="rs_subdiv_option_active_idx"), + ), + migrations.AddConstraint( + model_name="remotesensingsubdivisionoptionblock", + constraint=models.UniqueConstraint(fields=("option", "cluster_label"), name="rs_subdiv_option_block_unique_option_label"), + ), + migrations.AddConstraint( + model_name="remotesensingsubdivisionoptionassignment", + constraint=models.UniqueConstraint(fields=("option", "cell"), name="rs_subdiv_option_assign_unique_option_cell"), + ), + migrations.AddIndex( + model_name="remotesensingsubdivisionoptionassignment", + index=models.Index(fields=["option", "cluster_label"], name="rs_subopt_assign_lbl_idx"), + ), + ] diff --git a/location_data/migrations/0019_cluster_block_centers.py b/location_data/migrations/0019_cluster_block_centers.py new file mode 100644 index 0000000..2bf5520 --- /dev/null +++ b/location_data/migrations/0019_cluster_block_centers.py @@ -0,0 +1,81 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("location_data", "0018_remotesensingsubdivisionoption_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="remotesensingclusterblock", + name="center_cell_code", + field=models.CharField( + blank=True, + db_index=True, + default="", + help_text="شناسه سلول مرکزی انتخاب‌شده با بهینه‌سازی 1-center در همین کلاستر.", + max_length=64, + ), + ), + migrations.AddField( + model_name="remotesensingclusterblock", + name="center_cell_lat", + field=models.DecimalField( + blank=True, + db_index=True, + decimal_places=6, + help_text="عرض جغرافیایی سلول مرکزی کلاستر.", + max_digits=9, + null=True, + ), + ), + migrations.AddField( + model_name="remotesensingclusterblock", + name="center_cell_lon", + field=models.DecimalField( + blank=True, + db_index=True, + decimal_places=6, + help_text="طول جغرافیایی سلول مرکزی کلاستر.", + max_digits=9, + null=True, + ), + ), + migrations.AddField( + model_name="remotesensingsubdivisionoptionblock", + name="center_cell_code", + field=models.CharField( + blank=True, + db_index=True, + default="", + help_text="شناسه سلول مرکزی انتخاب‌شده با بهینه‌سازی 1-center روی اعضای همین کلاستر.", + max_length=64, + ), + ), + migrations.AddField( + model_name="remotesensingsubdivisionoptionblock", + name="center_cell_lat", + field=models.DecimalField( + blank=True, + db_index=True, + decimal_places=6, + help_text="عرض جغرافیایی سلول مرکزی کلاستر.", + max_digits=9, + null=True, + ), + ), + migrations.AddField( + model_name="remotesensingsubdivisionoptionblock", + name="center_cell_lon", + field=models.DecimalField( + blank=True, + db_index=True, + decimal_places=6, + help_text="طول جغرافیایی سلول مرکزی کلاستر.", + max_digits=9, + null=True, + ), + ), + ] diff --git a/location_data/models.py b/location_data/models.py index 2f87b7e..8d290d4 100644 --- a/location_data/models.py +++ b/location_data/models.py @@ -1,3 +1,5 @@ +import uuid + from django.db import models @@ -451,6 +453,252 @@ class RemoteSensingSubdivisionResult(models.Model): ) +class RemoteSensingSubdivisionOption(models.Model): + result = models.ForeignKey( + RemoteSensingSubdivisionResult, + on_delete=models.CASCADE, + related_name="options", + ) + requested_k = models.PositiveIntegerField(db_index=True) + effective_cluster_count = models.PositiveIntegerField(default=0) + is_active = models.BooleanField(default=False, db_index=True) + is_recommended = models.BooleanField(default=False, db_index=True) + selection_source = models.CharField( + max_length=32, + default="system", + help_text="منشا انتخاب این گزینه؛ مثل system یا user.", + ) + metadata = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["result", "requested_k", "id"] + constraints = [ + models.UniqueConstraint( + fields=["result", "requested_k"], + name="rs_subdiv_option_unique_result_requested_k", + ) + ] + indexes = [ + models.Index( + fields=["result", "is_active"], + name="rs_subdiv_option_active_idx", + ) + ] + verbose_name = "remote sensing subdivision option" + verbose_name_plural = "remote sensing subdivision options" + + def __str__(self): + return ( + f"RemoteSensingSubdivisionOption(result={self.result_id}, " + f"requested_k={self.requested_k}, active={self.is_active})" + ) + + +class RemoteSensingSubdivisionOptionBlock(models.Model): + option = models.ForeignKey( + RemoteSensingSubdivisionOption, + on_delete=models.CASCADE, + related_name="cluster_blocks", + ) + cluster_label = models.PositiveIntegerField(db_index=True) + sub_block_code = models.CharField(max_length=64, db_index=True) + chunk_size_sqm = models.PositiveIntegerField(default=900) + centroid_lat = models.DecimalField(max_digits=9, decimal_places=6, db_index=True) + centroid_lon = models.DecimalField(max_digits=9, decimal_places=6, db_index=True) + center_cell_code = models.CharField( + max_length=64, + blank=True, + default="", + db_index=True, + help_text="شناسه سلول مرکزی انتخاب‌شده با بهینه‌سازی 1-center روی اعضای همین کلاستر.", + ) + center_cell_lat = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + db_index=True, + help_text="عرض جغرافیایی سلول مرکزی کلاستر.", + ) + center_cell_lon = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + db_index=True, + help_text="طول جغرافیایی سلول مرکزی کلاستر.", + ) + geometry = models.JSONField(default=dict, blank=True) + cell_count = models.PositiveIntegerField(default=0) + cell_codes = models.JSONField(default=list, blank=True) + metadata = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["option", "cluster_label", "id"] + constraints = [ + models.UniqueConstraint( + fields=["option", "cluster_label"], + name="rs_subdiv_option_block_unique_option_label", + ) + ] + verbose_name = "remote sensing subdivision option block" + verbose_name_plural = "remote sensing subdivision option blocks" + + def __str__(self): + return ( + f"RemoteSensingSubdivisionOptionBlock(option={self.option_id}, " + f"cluster={self.cluster_label})" + ) + + +class RemoteSensingSubdivisionOptionAssignment(models.Model): + option = models.ForeignKey( + RemoteSensingSubdivisionOption, + on_delete=models.CASCADE, + related_name="assignments", + ) + cell = models.ForeignKey( + AnalysisGridCell, + on_delete=models.CASCADE, + related_name="subdivision_option_assignments", + ) + cluster_label = models.PositiveIntegerField(db_index=True) + raw_feature_values = models.JSONField(default=dict, blank=True) + scaled_feature_values = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["option", "cluster_label", "cell__cell_code"] + constraints = [ + models.UniqueConstraint( + fields=["option", "cell"], + name="rs_subdiv_option_assign_unique_option_cell", + ) + ] + indexes = [ + models.Index( + fields=["option", "cluster_label"], + name="rs_subopt_assign_lbl_idx", + ) + ] + verbose_name = "remote sensing subdivision option assignment" + verbose_name_plural = "remote sensing subdivision option assignments" + + def __str__(self): + return ( + f"RemoteSensingSubdivisionOptionAssignment(option={self.option_id}, " + f"cell={self.cell_id}, cluster={self.cluster_label})" + ) + + +class RemoteSensingClusterBlock(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True, db_index=True) + result = models.ForeignKey( + RemoteSensingSubdivisionResult, + on_delete=models.CASCADE, + related_name="cluster_blocks", + ) + soil_location = models.ForeignKey( + SoilLocation, + on_delete=models.CASCADE, + related_name="remote_sensing_cluster_blocks", + ) + block_subdivision = models.ForeignKey( + BlockSubdivision, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="remote_sensing_cluster_blocks", + ) + block_code = models.CharField( + max_length=64, + blank=True, + default="", + db_index=True, + help_text="شناسه بلوک والد که این زیر‌بلاک KMeans داخل آن ساخته شده است.", + ) + sub_block_code = models.CharField( + max_length=64, + db_index=True, + help_text="شناسه زیر‌بلاک ساخته‌شده توسط KMeans مثل cluster-0.", + ) + cluster_label = models.PositiveIntegerField(db_index=True) + chunk_size_sqm = models.PositiveIntegerField(default=900) + centroid_lat = models.DecimalField( + max_digits=9, + decimal_places=6, + db_index=True, + help_text="عرض جغرافیایی مرکز زیر‌بلاک.", + ) + centroid_lon = models.DecimalField( + max_digits=9, + decimal_places=6, + db_index=True, + help_text="طول جغرافیایی مرکز زیر‌بلاک.", + ) + center_cell_code = models.CharField( + max_length=64, + blank=True, + default="", + db_index=True, + help_text="شناسه سلول مرکزی انتخاب‌شده با بهینه‌سازی 1-center در همین کلاستر.", + ) + center_cell_lat = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + db_index=True, + help_text="عرض جغرافیایی سلول مرکزی کلاستر.", + ) + center_cell_lon = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + db_index=True, + help_text="طول جغرافیایی سلول مرکزی کلاستر.", + ) + geometry = models.JSONField( + default=dict, + blank=True, + help_text="هندسه GeoJSON زیر‌بلاک KMeans. فعلا از چندضلعی/چندچندضلعی سلول‌های عضو ساخته می‌شود.", + ) + cell_count = models.PositiveIntegerField(default=0) + cell_codes = models.JSONField(default=list, blank=True) + metadata = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["result", "cluster_label", "id"] + constraints = [ + models.UniqueConstraint( + fields=["result", "cluster_label"], + name="rs_cluster_block_unique_result_label", + ) + ] + indexes = [ + models.Index( + fields=["soil_location", "block_code", "cluster_label"], + name="rs_cluster_block_lookup_idx", + ) + ] + verbose_name = "remote sensing cluster block" + verbose_name_plural = "remote sensing cluster blocks" + + def __str__(self): + return ( + f"RemoteSensingClusterBlock({self.uuid}, " + f"{self.block_code or 'farm'}:{self.sub_block_code})" + ) + + class RemoteSensingClusterAssignment(models.Model): result = models.ForeignKey( RemoteSensingSubdivisionResult, diff --git a/location_data/openeo_service.py b/location_data/openeo_service.py index 0896759..c6ea67e 100644 --- a/location_data/openeo_service.py +++ b/location_data/openeo_service.py @@ -411,8 +411,7 @@ def build_spatial_extent(cells: list[AnalysisGridCell]) -> dict[str, float]: south = None north = None for cell in cells: - coordinates = ((cell.geometry or {}).get("coordinates") or [[]])[0] - for lon, lat in coordinates: + for lon, lat in _iter_geometry_lon_lat_pairs(cell.geometry): west = lon if west is None else min(west, lon) east = lon if east is None else max(east, lon) south = lat if south is None else min(south, lat) @@ -426,6 +425,27 @@ def build_spatial_extent(cells: list[AnalysisGridCell]) -> dict[str, float]: } +def _iter_geometry_lon_lat_pairs(geometry: dict[str, Any] | None): + geometry = dict(geometry or {}) + geometry_type = geometry.get("type") + coordinates = geometry.get("coordinates") or [] + + if geometry_type == "Polygon": + for ring in coordinates: + for point in ring or []: + if len(point) >= 2: + yield point[0], point[1] + return + + if geometry_type == "MultiPolygon": + for polygon in coordinates: + for ring in polygon or []: + for point in ring or []: + if len(point) >= 2: + yield point[0], point[1] + return + + def build_empty_metric_payload() -> dict[str, Any]: return {metric_name: None for metric_name in METRIC_NAMES} diff --git a/location_data/serializers.py b/location_data/serializers.py index 36697bb..07a396e 100644 --- a/location_data/serializers.py +++ b/location_data/serializers.py @@ -3,9 +3,12 @@ from rest_framework import serializers from .models import ( AnalysisGridObservation, BlockSubdivision, + RemoteSensingClusterBlock, RemoteSensingRun, RemoteSensingClusterAssignment, RemoteSensingSubdivisionResult, + RemoteSensingSubdivisionOption, + RemoteSensingSubdivisionOptionBlock, SoilLocation, ) from .satellite_snapshot import build_location_block_satellite_snapshots @@ -141,6 +144,25 @@ class RemoteSensingFarmRequestSerializer(serializers.Serializer): page_size = serializers.IntegerField(required=False, min_value=1, max_value=200, default=100) +class RemoteSensingClusterBlockLiveRequestSerializer(serializers.Serializer): + temporal_start = serializers.DateField(required=False) + temporal_end = serializers.DateField(required=False) + days = serializers.IntegerField(required=False, min_value=1, max_value=90, default=30) + + def validate(self, attrs): + temporal_start = attrs.get("temporal_start") + temporal_end = attrs.get("temporal_end") + if bool(temporal_start) != bool(temporal_end): + raise serializers.ValidationError( + "برای بازه سفارشی باید هر دو فیلد temporal_start و temporal_end ارسال شوند." + ) + if temporal_start and temporal_end and temporal_start > temporal_end: + raise serializers.ValidationError( + {"temporal_start": ["temporal_start نمی‌تواند بعد از temporal_end باشد."]} + ) + return attrs + + class RemoteSensingCellObservationSerializer(serializers.ModelSerializer): cell_code = serializers.CharField(source="cell.cell_code", read_only=True) block_code = serializers.CharField(source="cell.block_code", read_only=True) @@ -175,6 +197,13 @@ class RemoteSensingSummarySerializer(serializers.Serializer): soil_vv_db_mean = serializers.FloatField(allow_null=True) +class RemoteSensingClusterBlockLiveMetricsSerializer(serializers.Serializer): + ndvi = serializers.FloatField(allow_null=True) + ndwi = serializers.FloatField(allow_null=True) + soil_vv = serializers.FloatField(allow_null=True) + soil_vv_db = serializers.FloatField(allow_null=True) + + class RemoteSensingRunSerializer(serializers.ModelSerializer): status_label = serializers.SerializerMethodField() pipeline_status = serializers.SerializerMethodField() @@ -239,8 +268,74 @@ class RemoteSensingClusterAssignmentSerializer(serializers.ModelSerializer): ] +class RemoteSensingClusterBlockSerializer(serializers.ModelSerializer): + class Meta: + model = RemoteSensingClusterBlock + fields = [ + "uuid", + "sub_block_code", + "cluster_label", + "chunk_size_sqm", + "centroid_lat", + "centroid_lon", + "center_cell_code", + "center_cell_lat", + "center_cell_lon", + "cell_count", + "cell_codes", + "geometry", + "metadata", + "created_at", + "updated_at", + ] + + +class RemoteSensingSubdivisionOptionBlockSerializer(serializers.ModelSerializer): + class Meta: + model = RemoteSensingSubdivisionOptionBlock + fields = [ + "cluster_label", + "sub_block_code", + "chunk_size_sqm", + "centroid_lat", + "centroid_lon", + "center_cell_code", + "center_cell_lat", + "center_cell_lon", + "cell_count", + "cell_codes", + "geometry", + "metadata", + ] + + +class RemoteSensingSubdivisionOptionSerializer(serializers.ModelSerializer): + cluster_blocks = RemoteSensingSubdivisionOptionBlockSerializer(many=True, read_only=True) + + class Meta: + model = RemoteSensingSubdivisionOption + fields = [ + "id", + "requested_k", + "effective_cluster_count", + "is_active", + "is_recommended", + "selection_source", + "metadata", + "cluster_blocks", + "created_at", + "updated_at", + ] + + +class RemoteSensingSubdivisionOptionActivateSerializer(serializers.Serializer): + requested_k = serializers.IntegerField(min_value=1) + + class RemoteSensingSubdivisionResultSerializer(serializers.ModelSerializer): assignments = serializers.SerializerMethodField() + cluster_blocks = RemoteSensingClusterBlockSerializer(many=True, read_only=True) + available_k_options = serializers.SerializerMethodField() def get_assignments(self, obj): assignments = self.context.get("paginated_assignments") @@ -248,6 +343,10 @@ class RemoteSensingSubdivisionResultSerializer(serializers.ModelSerializer): assignments = obj.assignments.all().order_by("cluster_label", "cell__cell_code") return RemoteSensingClusterAssignmentSerializer(assignments, many=True).data + def get_available_k_options(self, obj): + options = obj.options.all().order_by("requested_k") + return RemoteSensingSubdivisionOptionSerializer(options, many=True).data + class Meta: model = RemoteSensingSubdivisionResult fields = [ @@ -260,6 +359,8 @@ class RemoteSensingSubdivisionResultSerializer(serializers.ModelSerializer): "selected_features", "skipped_cell_codes", "metadata", + "available_k_options", + "cluster_blocks", "assignments", "created_at", "updated_at", @@ -309,3 +410,30 @@ class RemoteSensingRunResultResponseSerializer(serializers.Serializer): run = RemoteSensingRunSerializer() subdivision_result = RemoteSensingSubdivisionResultSerializer(allow_null=True) pagination = serializers.JSONField(required=False) + + +class RemoteSensingClusterBlockLiveResponseSerializer(serializers.Serializer): + status = serializers.CharField() + source = serializers.CharField() + cluster_block = RemoteSensingClusterBlockSerializer() + temporal_extent = serializers.JSONField() + selected_features = serializers.ListField( + child=serializers.CharField(), + allow_empty=False, + ) + summary = RemoteSensingSummarySerializer() + metrics = RemoteSensingClusterBlockLiveMetricsSerializer() + metadata = serializers.JSONField() + + +class RemoteSensingSubdivisionOptionListResponseSerializer(serializers.Serializer): + result_id = serializers.IntegerField() + active_requested_k = serializers.IntegerField(allow_null=True) + recommended_requested_k = serializers.IntegerField(allow_null=True) + options = RemoteSensingSubdivisionOptionSerializer(many=True) + + +class RemoteSensingSubdivisionOptionActivateResponseSerializer(serializers.Serializer): + result_id = serializers.IntegerField() + activated_requested_k = serializers.IntegerField() + subdivision_result = RemoteSensingSubdivisionResultSerializer() diff --git a/location_data/test_cluster_block_live_api.py b/location_data/test_cluster_block_live_api.py new file mode 100644 index 0000000..2137f0f --- /dev/null +++ b/location_data/test_cluster_block_live_api.py @@ -0,0 +1,195 @@ +from datetime import date, timedelta +import uuid +from unittest.mock import patch + +from django.test import TestCase, override_settings +from django.utils import timezone +from rest_framework.test import APIClient + +from location_data.models import ( + AnalysisGridCell, + BlockSubdivision, + RemoteSensingClusterBlock, + RemoteSensingRun, + RemoteSensingSubdivisionResult, + SoilLocation, +) + + +@override_settings(ROOT_URLCONF="location_data.urls") +class RemoteSensingClusterBlockLiveApiTests(TestCase): + def setUp(self): + self.client = APIClient() + self.boundary = { + "type": "Polygon", + "coordinates": [ + [ + [51.3890, 35.6890], + [51.3900, 35.6890], + [51.3900, 35.6900], + [51.3890, 35.6900], + [51.3890, 35.6890], + ] + ], + } + self.location = SoilLocation.objects.create( + latitude="35.689200", + longitude="51.389000", + farm_boundary=self.boundary, + ) + self.subdivision = BlockSubdivision.objects.create( + soil_location=self.location, + block_code="block-1", + source_boundary=self.boundary, + chunk_size_sqm=900, + status="subdivided", + ) + self.run = RemoteSensingRun.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + chunk_size_sqm=900, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + status=RemoteSensingRun.STATUS_SUCCESS, + metadata={"stage": "completed"}, + ) + self.result = RemoteSensingSubdivisionResult.objects.create( + soil_location=self.location, + run=self.run, + block_subdivision=self.subdivision, + block_code="block-1", + chunk_size_sqm=900, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + cluster_count=1, + selected_features=["ndvi", "ndwi", "soil_vv_db"], + metadata={"used_cell_count": 2, "skipped_cell_count": 0}, + ) + self.cluster_block = RemoteSensingClusterBlock.objects.create( + result=self.result, + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + sub_block_code="cluster-0", + cluster_label=0, + chunk_size_sqm=900, + centroid_lat="35.689500", + centroid_lon="51.389500", + cell_count=2, + cell_codes=["cell-1", "cell-2"], + geometry=self.boundary, + metadata={"source": "analysis_grid_cells"}, + ) + + def test_get_cluster_block_live_returns_404_when_uuid_missing(self): + response = self.client.get( + f"/remote-sensing/cluster-blocks/{uuid.uuid4()}/live/" + ) + + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["msg"], "زیر‌بلاک KMeans پیدا نشد.") + + @patch("location_data.views.compute_remote_sensing_metrics") + def test_get_cluster_block_live_reads_fresh_metrics_from_openeo(self, compute_mock): + compute_mock.return_value = { + "results": { + f"cluster-{self.cluster_block.uuid}": { + "ndvi": 0.63, + "ndwi": 0.21, + "soil_vv": 0.13, + "soil_vv_db": -8.860566, + } + }, + "metadata": { + "backend": "openeo", + "backend_url": "https://openeofed.dataspace.copernicus.eu", + "collections_used": ["SENTINEL2_L2A", "SENTINEL1_GRD"], + "job_refs": {"ndvi": "job-1", "ndwi": "job-2", "soil_vv": "job-3"}, + "failed_metrics": [], + "payload_diagnostics": {}, + }, + } + + response = self.client.get( + f"/remote-sensing/cluster-blocks/{self.cluster_block.uuid}/live/", + data={"temporal_start": "2025-02-01", "temporal_end": "2025-02-15"}, + ) + + self.assertEqual(response.status_code, 200) + payload = response.json()["data"] + self.assertEqual(payload["status"], "success") + self.assertEqual(payload["source"], "openeo") + self.assertEqual(payload["cluster_block"]["uuid"], str(self.cluster_block.uuid)) + self.assertEqual(payload["summary"]["cell_count"], 2) + self.assertEqual(payload["summary"]["ndvi_mean"], 0.63) + self.assertEqual(payload["metrics"]["soil_vv_db"], -8.860566) + self.assertEqual(payload["temporal_extent"]["start_date"], "2025-02-01") + self.assertEqual(payload["temporal_extent"]["end_date"], "2025-02-15") + self.assertEqual( + payload["metadata"]["requested_cluster_uuid"], + str(self.cluster_block.uuid), + ) + + compute_mock.assert_called_once() + args, kwargs = compute_mock.call_args + self.assertEqual(len(args[0]), 1) + self.assertEqual(args[0][0].geometry, self.boundary) + self.assertEqual(kwargs["temporal_start"].isoformat(), "2025-02-01") + self.assertEqual(kwargs["temporal_end"].isoformat(), "2025-02-15") + + @patch("location_data.views.compute_remote_sensing_metrics") + def test_get_cluster_block_live_rebuilds_geometry_from_member_cells_when_missing(self, compute_mock): + cell_geometry = { + "type": "Polygon", + "coordinates": [ + [ + [51.3890, 35.6890], + [51.3895, 35.6890], + [51.3895, 35.6895], + [51.3890, 35.6895], + [51.3890, 35.6890], + ] + ], + } + AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code="cell-1", + chunk_size_sqm=900, + geometry=cell_geometry, + centroid_lat="35.689250", + centroid_lon="51.389250", + ) + self.cluster_block.geometry = {} + self.cluster_block.save(update_fields=["geometry", "updated_at"]) + compute_mock.return_value = { + "results": { + f"cluster-{self.cluster_block.uuid}": { + "ndvi": 0.55, + "ndwi": 0.18, + "soil_vv": 0.10, + "soil_vv_db": -10.0, + } + }, + "metadata": {"job_refs": {}, "failed_metrics": [], "payload_diagnostics": {}}, + } + + response = self.client.get( + f"/remote-sensing/cluster-blocks/{self.cluster_block.uuid}/live/", + data={"days": 7}, + ) + + self.assertEqual(response.status_code, 200) + payload = response.json()["data"] + self.assertEqual(payload["metrics"]["ndvi"], 0.55) + + compute_mock.assert_called_once() + args, kwargs = compute_mock.call_args + self.assertEqual(args[0][0].geometry["type"], "Polygon") + self.assertEqual(args[0][0].geometry, cell_geometry) + expected_end = timezone.localdate() - timedelta(days=1) + expected_start = expected_end - timedelta(days=6) + self.assertEqual(kwargs["temporal_start"], expected_start) + self.assertEqual(kwargs["temporal_end"], expected_end) diff --git a/location_data/test_data_driven_subdivision.py b/location_data/test_data_driven_subdivision.py index 01badd9..620c8f3 100644 --- a/location_data/test_data_driven_subdivision.py +++ b/location_data/test_data_driven_subdivision.py @@ -7,17 +7,25 @@ from django.core.files.base import ContentFile from django.test import TestCase from location_data.data_driven_subdivision import ( + ClusteringDataset, EmptyObservationDatasetError, _persist_remote_sensing_diagnostic_artifacts, + _build_observation_label, + _build_cluster_geometry, + build_cluster_summaries, build_clustering_dataset, + create_remote_sensing_subdivision_result, + enforce_spatial_contiguity, sync_block_subdivision_with_result, ) from location_data.models import ( AnalysisGridCell, AnalysisGridObservation, BlockSubdivision, + RemoteSensingClusterBlock, RemoteSensingRun, RemoteSensingSubdivisionResult, + RemoteSensingSubdivisionOption, SoilLocation, ) @@ -58,7 +66,8 @@ class DataDrivenSubdivisionSyncTests(TestCase): status=RemoteSensingRun.STATUS_SUCCESS, ) - def test_sync_block_subdivision_with_result_updates_saved_sub_blocks(self): + @patch("location_data.data_driven_subdivision.render_elbow_plot", return_value=None) + def test_sync_block_subdivision_with_result_updates_saved_sub_blocks(self, _mock_plot): cell_1 = AnalysisGridCell.objects.create( soil_location=self.location, block_subdivision=self.subdivision, @@ -190,6 +199,9 @@ class DataDrivenSubdivisionSyncTests(TestCase): ), patch( "location_data.data_driven_subdivision._render_feature_pair_plot", return_value=ContentFile(b"pairs"), + ), patch( + "location_data.data_driven_subdivision._render_feature_projection_plot", + return_value=ContentFile(b"projection"), ): artifacts = _persist_remote_sensing_diagnostic_artifacts( result=result, @@ -207,13 +219,23 @@ class DataDrivenSubdivisionSyncTests(TestCase): selected_features=["ndvi", "ndwi", "soil_vv_db"], scaled_matrix=[[0.0, 0.0, 0.0]], inertia_curve=[{"k": 1, "sse": 0.0}], + requested_k=1, + effective_cluster_count=1, ) self.assertEqual( sorted(artifacts["files"].keys()), - ["cluster_map", "cluster_sizes", "elbow_plot", "feature_pairs"], + [ + "cluster_map", + "cluster_sizes", + "elbow_plot", + "feature_pairs", + "feature_projection", + ], ) + self.assertIn("k-1-effective-1", artifacts["directory"]) for path in artifacts["files"].values(): self.assertTrue(os.path.exists(path)) + self.assertIn("__k-1__effective-1__", path) def test_build_clustering_dataset_raises_clear_error_when_all_selected_features_are_null(self): cell = AnalysisGridCell.objects.create( @@ -250,3 +272,301 @@ class DataDrivenSubdivisionSyncTests(TestCase): self.assertIn("No usable observations available for clustering", joined) self.assertIn('"run_id": {}'.format(self.run.id), joined) self.assertIn('"region_id": {}'.format(self.location.id), joined) + + def test_build_cluster_summaries_selects_middle_grid_as_k_center(self): + observations = [] + for index in range(3): + cell = AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code=f"cell-{index}", + chunk_size_sqm=900, + geometry={ + "type": "Polygon", + "coordinates": [[ + [51.3890 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6890], + ]], + }, + centroid_lat="35.689200", + centroid_lon=f"{51.3892 + (index * 0.0001):.6f}", + ) + observations.append( + AnalysisGridObservation.objects.create( + cell=cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.2 + index, + ) + ) + + cluster_summaries = build_cluster_summaries( + observations=observations, + labels=[0, 0, 0], + ) + + self.assertEqual(cluster_summaries[0]["center_cell_code"], "cell-1") + self.assertEqual(cluster_summaries[0]["center_cell_lat"], 35.6892) + self.assertEqual(cluster_summaries[0]["center_cell_lon"], 51.3893) + + def test_build_observation_label_uses_numeric_index_for_30m_cells(self): + cell = AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code="cell-arbitrary-name", + chunk_size_sqm=900, + geometry=self.boundary, + centroid_lat="35.689200", + centroid_lon="51.389200", + ) + observation = AnalysisGridObservation.objects.create( + cell=cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.5, + ) + + self.assertEqual(_build_observation_label(observation=observation, index=0), "1") + self.assertEqual(_build_observation_label(observation=observation, index=7), "8") + + @patch("location_data.data_driven_subdivision.run_kmeans_labels", return_value=[0, 1, 1]) + @patch("location_data.data_driven_subdivision.choose_cluster_count", return_value=(2, [])) + @patch("location_data.data_driven_subdivision.build_clustering_dataset") + @patch("location_data.data_driven_subdivision._persist_remote_sensing_diagnostic_artifacts", return_value={}) + @patch("location_data.data_driven_subdivision.render_elbow_plot", return_value=None) + def test_create_remote_sensing_subdivision_result_persists_cluster_blocks_with_geometry( + self, + _mock_plot, + _mock_artifacts, + mock_build_dataset, + _mock_choose_k, + _mock_run_kmeans, + ): + cells = [ + AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code=f"cell-{index}", + chunk_size_sqm=900, + geometry={ + "type": "Polygon", + "coordinates": [[ + [51.3890 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6890], + ]], + }, + centroid_lat=f"{35.6892 + (index * 0.0001):.6f}", + centroid_lon=f"{51.3892 + (index * 0.0001):.6f}", + ) + for index in range(3) + ] + observations = [ + AnalysisGridObservation.objects.create( + cell=cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.2 + (index * 0.3), + ndwi=0.1 + (index * 0.2), + soil_vv_db=-8.0 + index, + ) + for index, cell in enumerate(cells) + ] + mock_build_dataset.return_value = ClusteringDataset( + observations=observations, + selected_features=["ndvi", "ndwi", "soil_vv_db"], + raw_feature_rows=[[0.2, 0.1, -8.0], [0.5, 0.3, -7.0], [0.8, 0.5, -6.0]], + raw_feature_maps=[ + {"ndvi": 0.2, "ndwi": 0.1, "soil_vv_db": -8.0}, + {"ndvi": 0.5, "ndwi": 0.3, "soil_vv_db": -7.0}, + {"ndvi": 0.8, "ndwi": 0.5, "soil_vv_db": -6.0}, + ], + skipped_cell_codes=[], + used_cell_codes=[cell.cell_code for cell in cells], + imputed_matrix=[[0.2, 0.1, -8.0], [0.5, 0.3, -7.0], [0.8, 0.5, -6.0]], + scaled_matrix=[[-1.0, -1.0, -1.0], [0.0, 0.0, 0.0], [1.0, 1.0, 1.0]], + imputer_statistics={"ndvi": 0.5, "ndwi": 0.3, "soil_vv_db": -7.0}, + scaler_means={"ndvi": 0.5, "ndwi": 0.3, "soil_vv_db": -7.0}, + scaler_scales={"ndvi": 0.1, "ndwi": 0.1, "soil_vv_db": 1.0}, + missing_value_counts={"ndvi": 0, "ndwi": 0, "soil_vv_db": 0}, + skipped_reasons={"all_features_missing": []}, + ) + + result = create_remote_sensing_subdivision_result( + location=self.location, + run=self.run, + observations=observations, + block_subdivision=self.subdivision, + block_code="block-1", + selected_features=["ndvi", "ndwi", "soil_vv_db"], + explicit_k=2, + ) + + self.assertEqual(_mock_artifacts.call_count, 4) + requested_ks = sorted( + { + call.kwargs.get("requested_k") + for call in _mock_artifacts.call_args_list + if call.kwargs.get("requested_k") is not None + } + ) + self.assertEqual(requested_ks, [1, 2, 3]) + + cluster_blocks = list(result.cluster_blocks.order_by("cluster_label")) + self.assertEqual(len(cluster_blocks), 2) + self.assertTrue(all(cluster_block.uuid for cluster_block in cluster_blocks)) + self.assertTrue(all(cluster_block.geometry for cluster_block in cluster_blocks)) + self.assertEqual( + sum(cluster_block.cell_count for cluster_block in cluster_blocks), + 3, + ) + self.assertEqual(RemoteSensingClusterBlock.objects.filter(result=result).count(), 2) + + result.refresh_from_db() + cluster_summaries = result.metadata["cluster_summaries"] + self.assertTrue(all(summary.get("cluster_uuid") for summary in cluster_summaries)) + self.assertTrue(all(summary.get("geometry") for summary in cluster_summaries)) + self.assertEqual(cluster_summaries[0]["center_cell_code"], "cell-0") + self.assertEqual(cluster_summaries[1]["center_cell_code"], "cell-1") + self.assertEqual(result.cluster_count, 2) + self.assertEqual( + result.metadata["spatial_constraint"]["final_cluster_count"], + 2, + ) + self.assertEqual( + list( + RemoteSensingSubdivisionOption.objects.filter(result=result) + .order_by("requested_k") + .values_list("requested_k", flat=True) + ), + [1, 2, 3], + ) + self.assertEqual(result.options.filter(is_active=True).get().requested_k, 2) + self.assertEqual(result.options.filter(is_recommended=True).get().requested_k, 2) + + self.subdivision.refresh_from_db() + self.assertTrue(all(point.get("cluster_uuid") for point in self.subdivision.centroid_points)) + self.assertEqual(self.subdivision.centroid_points[1]["center_cell_code"], "cell-1") + + self.location.refresh_from_db() + block_layout = self.location.block_layout["blocks"][0] + self.assertTrue(all(block.get("cluster_uuid") for block in block_layout["sub_blocks"])) + self.assertEqual(block_layout["sub_blocks"][1]["center_cell_code"], "cell-1") + self.assertEqual(cluster_blocks[1].geometry["type"], "Polygon") + self.assertEqual(cluster_blocks[1].center_cell_code, "cell-1") + + def test_enforce_spatial_contiguity_merges_diagonal_island_into_adjacent_cluster(self): + cell_payloads = [ + ("cell-00", [[51.3890, 35.6890], [51.3891, 35.6890], [51.3891, 35.6891], [51.3890, 35.6891], [51.3890, 35.6890]]), + ("cell-01", [[51.3891, 35.6890], [51.3892, 35.6890], [51.3892, 35.6891], [51.3891, 35.6891], [51.3891, 35.6890]]), + ("cell-10", [[51.3890, 35.6891], [51.3891, 35.6891], [51.3891, 35.6892], [51.3890, 35.6892], [51.3890, 35.6891]]), + ("cell-11", [[51.3891, 35.6891], [51.3892, 35.6891], [51.3892, 35.6892], [51.3891, 35.6892], [51.3891, 35.6891]]), + ] + observations = [] + for index, (cell_code, ring) in enumerate(cell_payloads): + cell = AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code=cell_code, + chunk_size_sqm=900, + geometry={"type": "Polygon", "coordinates": [ring]}, + centroid_lat=f"{35.68905 + (index // 2) * 0.0001:.6f}", + centroid_lon=f"{51.38905 + (index % 2) * 0.0001:.6f}", + ) + observations.append( + AnalysisGridObservation.objects.create( + cell=cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.1 + index, + ) + ) + + labels, metadata = enforce_spatial_contiguity( + observations=observations, + labels=[0, 1, 1, 0], + scaled_matrix=[ + [0.0, 0.0, 0.0], + [1.0, 1.0, 1.0], + [1.1, 1.1, 1.1], + [0.1, 0.1, 0.1], + ], + ) + + self.assertEqual(labels, [0, 1, 1, 1]) + self.assertTrue(metadata["applied"]) + self.assertEqual(metadata["disconnected_components_merged"], 1) + + def test_build_cluster_geometry_returns_single_polygon_for_adjacent_cells(self): + left_cell = AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code="cell-left", + chunk_size_sqm=900, + geometry={ + "type": "Polygon", + "coordinates": [[ + [51.3890, 35.6890], + [51.3891, 35.6890], + [51.3891, 35.6891], + [51.3890, 35.6891], + [51.3890, 35.6890], + ]], + }, + centroid_lat="35.689050", + centroid_lon="51.389050", + ) + right_cell = AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code="cell-right", + chunk_size_sqm=900, + geometry={ + "type": "Polygon", + "coordinates": [[ + [51.3891, 35.6890], + [51.3892, 35.6890], + [51.3892, 35.6891], + [51.3891, 35.6891], + [51.3891, 35.6890], + ]], + }, + centroid_lat="35.689050", + centroid_lon="51.389150", + ) + observations = [ + AnalysisGridObservation.objects.create( + cell=left_cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.4, + ), + AnalysisGridObservation.objects.create( + cell=right_cell, + run=self.run, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + ndvi=0.5, + ), + ] + + geometry = _build_cluster_geometry(observations) + + self.assertEqual(geometry["type"], "Polygon") + self.assertEqual(len(geometry["coordinates"][0]), 7) diff --git a/location_data/test_k_options_api.py b/location_data/test_k_options_api.py new file mode 100644 index 0000000..eba042d --- /dev/null +++ b/location_data/test_k_options_api.py @@ -0,0 +1,207 @@ +from datetime import date +from unittest.mock import patch + +from django.test import TestCase, override_settings +from rest_framework.test import APIClient + +from location_data.models import ( + AnalysisGridCell, + BlockSubdivision, + RemoteSensingSubdivisionOption, + RemoteSensingSubdivisionOptionAssignment, + RemoteSensingSubdivisionOptionBlock, + RemoteSensingRun, + RemoteSensingSubdivisionResult, + SoilLocation, +) + + +@override_settings(ROOT_URLCONF="location_data.urls") +class RemoteSensingSubdivisionOptionApiTests(TestCase): + def setUp(self): + self.client = APIClient() + self.boundary = { + "type": "Polygon", + "coordinates": [ + [ + [51.3890, 35.6890], + [51.3900, 35.6890], + [51.3900, 35.6900], + [51.3890, 35.6900], + [51.3890, 35.6890], + ] + ], + } + self.location = SoilLocation.objects.create( + latitude="35.689200", + longitude="51.389000", + farm_boundary=self.boundary, + ) + self.subdivision = BlockSubdivision.objects.create( + soil_location=self.location, + block_code="block-1", + source_boundary=self.boundary, + chunk_size_sqm=900, + status="subdivided", + ) + self.run = RemoteSensingRun.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + chunk_size_sqm=900, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + status=RemoteSensingRun.STATUS_SUCCESS, + metadata={"stage": "completed"}, + ) + self.result = RemoteSensingSubdivisionResult.objects.create( + soil_location=self.location, + run=self.run, + block_subdivision=self.subdivision, + block_code="block-1", + chunk_size_sqm=900, + temporal_start=date(2025, 1, 1), + temporal_end=date(2025, 1, 31), + cluster_count=1, + selected_features=["ndvi", "ndwi", "soil_vv_db"], + metadata={"recommended_requested_k": 2, "active_requested_k": 1}, + ) + self.cells = [ + AnalysisGridCell.objects.create( + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="block-1", + cell_code=f"cell-{index}", + chunk_size_sqm=900, + geometry={ + "type": "Polygon", + "coordinates": [[ + [51.3890 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6890], + [51.3891 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6891], + [51.3890 + (index * 0.0001), 35.6890], + ]], + }, + centroid_lat=f"{35.68905 + (index * 0.0001):.6f}", + centroid_lon=f"{51.38905 + (index * 0.0001):.6f}", + ) + for index in range(2) + ] + self.option_k1 = RemoteSensingSubdivisionOption.objects.create( + result=self.result, + requested_k=1, + effective_cluster_count=1, + is_active=True, + is_recommended=False, + selection_source="system", + metadata={"cluster_summaries": []}, + ) + self.option_k2 = RemoteSensingSubdivisionOption.objects.create( + result=self.result, + requested_k=2, + effective_cluster_count=2, + is_active=False, + is_recommended=True, + selection_source="system", + metadata={"cluster_summaries": []}, + ) + for cell in self.cells: + RemoteSensingSubdivisionOptionAssignment.objects.create( + option=self.option_k1, + cell=cell, + cluster_label=0, + raw_feature_values={"ndvi": 0.4}, + scaled_feature_values={"ndvi": 0.0}, + ) + RemoteSensingSubdivisionOptionBlock.objects.create( + option=self.option_k1, + cluster_label=0, + sub_block_code="cluster-0", + chunk_size_sqm=900, + centroid_lat="35.689100", + centroid_lon="51.389100", + center_cell_code="cell-0", + center_cell_lat="35.689050", + center_cell_lon="51.389050", + cell_count=2, + cell_codes=[cell.cell_code for cell in self.cells], + geometry=self.boundary, + metadata={ + "source": "analysis_grid_cells", + "center_selection": {"strategy": "coordinate_1_center", "center_cell_code": "cell-0"}, + }, + ) + for index, cell in enumerate(self.cells): + RemoteSensingSubdivisionOptionAssignment.objects.create( + option=self.option_k2, + cell=cell, + cluster_label=index, + raw_feature_values={"ndvi": 0.4 + index}, + scaled_feature_values={"ndvi": float(index)}, + ) + RemoteSensingSubdivisionOptionBlock.objects.create( + option=self.option_k2, + cluster_label=index, + sub_block_code=f"cluster-{index}", + chunk_size_sqm=900, + centroid_lat=f"{35.68905 + (index * 0.0001):.6f}", + centroid_lon=f"{51.38905 + (index * 0.0001):.6f}", + center_cell_code=cell.cell_code, + center_cell_lat=f"{35.68905 + (index * 0.0001):.6f}", + center_cell_lon=f"{51.38905 + (index * 0.0001):.6f}", + cell_count=1, + cell_codes=[cell.cell_code], + geometry=cell.geometry, + metadata={ + "source": "analysis_grid_cells", + "center_selection": {"strategy": "coordinate_1_center", "center_cell_code": cell.cell_code}, + }, + ) + + def test_get_k_options_returns_all_persisted_options(self): + response = self.client.get( + f"/remote-sensing/results/{self.result.id}/k-options/" + ) + + self.assertEqual(response.status_code, 200) + payload = response.json()["data"] + self.assertEqual(payload["result_id"], self.result.id) + self.assertEqual(payload["active_requested_k"], 1) + self.assertEqual(payload["recommended_requested_k"], 2) + self.assertEqual([item["requested_k"] for item in payload["options"]], [1, 2]) + self.assertEqual(payload["options"][0]["cluster_blocks"][0]["center_cell_code"], "cell-0") + + @patch("location_data.data_driven_subdivision.render_elbow_plot", return_value=None) + def test_post_activate_k_marks_selected_option_active_and_syncs_result(self, _mock_plot): + response = self.client.post( + f"/remote-sensing/results/{self.result.id}/k-options/activate/", + data={"requested_k": 2}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + payload = response.json()["data"] + self.assertEqual(payload["activated_requested_k"], 2) + self.assertEqual(payload["subdivision_result"]["cluster_count"], 2) + self.assertEqual( + payload["subdivision_result"]["metadata"]["active_requested_k"], + 2, + ) + self.assertEqual(len(payload["subdivision_result"]["cluster_blocks"]), 2) + self.assertEqual( + payload["subdivision_result"]["cluster_blocks"][0]["center_cell_code"], + "cell-0", + ) + + self.option_k1.refresh_from_db() + self.option_k2.refresh_from_db() + self.assertFalse(self.option_k1.is_active) + self.assertTrue(self.option_k2.is_active) + self.assertEqual(self.option_k2.selection_source, "user") + + self.result.refresh_from_db() + self.assertEqual(self.result.cluster_count, 2) + self.assertEqual(self.result.assignments.count(), 2) + self.assertEqual(self.result.cluster_blocks.count(), 2) + self.assertEqual(self.result.cluster_blocks.order_by("cluster_label").first().center_cell_code, "cell-0") diff --git a/location_data/test_remote_sensing_api.py b/location_data/test_remote_sensing_api.py index a7c2ada..934d3fa 100644 --- a/location_data/test_remote_sensing_api.py +++ b/location_data/test_remote_sensing_api.py @@ -11,6 +11,7 @@ from location_data.data_driven_subdivision import DEFAULT_CLUSTER_FEATURES from location_data.models import ( AnalysisGridCell, AnalysisGridObservation, + RemoteSensingClusterBlock, BlockSubdivision, RemoteSensingClusterAssignment, RemoteSensingRun, @@ -457,6 +458,21 @@ class RemoteSensingApiTests(TestCase): raw_feature_values={"ndvi": 0.61}, scaled_feature_values={"ndvi": 0.0}, ) + cluster_block = RemoteSensingClusterBlock.objects.create( + result=result, + soil_location=self.location, + block_subdivision=self.subdivision, + block_code="", + sub_block_code="cluster-0", + cluster_label=0, + chunk_size_sqm=900, + centroid_lat="35.689500", + centroid_lon="51.389500", + cell_count=1, + cell_codes=["cell-1"], + geometry=self.boundary, + metadata={"source": "analysis_grid_cells"}, + ) task_id = "e723ba3e-c53c-401b-b3a0-5f7013c7b401" run.metadata = {**run.metadata, "task_id": task_id} @@ -468,5 +484,6 @@ class RemoteSensingApiTests(TestCase): payload = response.json()["data"] self.assertEqual(payload["status"], "completed") self.assertEqual(payload["subdivision_result"]["cluster_count"], 1) + self.assertEqual(payload["subdivision_result"]["cluster_blocks"][0]["uuid"], str(cluster_block.uuid)) self.assertEqual(len(payload["subdivision_result"]["assignments"]), 1) self.assertEqual(payload["pagination"]["assignments"]["total_items"], 1) diff --git a/location_data/urls.py b/location_data/urls.py index e32d62c..84bb8c4 100644 --- a/location_data/urls.py +++ b/location_data/urls.py @@ -3,6 +3,9 @@ from django.urls import path from .views import ( NdviHealthView, RemoteSensingAnalysisView, + RemoteSensingClusterBlockLiveView, + RemoteSensingSubdivisionOptionActivateView, + RemoteSensingSubdivisionOptionListView, RemoteSensingRunStatusView, SoilDataView, ) @@ -10,6 +13,21 @@ from .views import ( urlpatterns = [ path("", SoilDataView.as_view(), name="soil-data"), path("remote-sensing/", RemoteSensingAnalysisView.as_view(), name="remote-sensing"), + path( + "remote-sensing/cluster-blocks//live/", + RemoteSensingClusterBlockLiveView.as_view(), + name="remote-sensing-cluster-block-live", + ), + path( + "remote-sensing/results//k-options/", + RemoteSensingSubdivisionOptionListView.as_view(), + name="remote-sensing-k-options", + ), + path( + "remote-sensing/results//k-options/activate/", + RemoteSensingSubdivisionOptionActivateView.as_view(), + name="remote-sensing-k-options-activate", + ), path("remote-sensing/runs//status/", RemoteSensingRunStatusView.as_view(), name="remote-sensing-run-status"), path("ndvi-health/", NdviHealthView.as_view(), name="ndvi-health"), ] diff --git a/location_data/views.py b/location_data/views.py index d9d3bb8..2c19018 100644 --- a/location_data/views.py +++ b/location_data/views.py @@ -1,4 +1,5 @@ from datetime import timedelta +from types import SimpleNamespace from typing import Any from django.apps import apps @@ -26,13 +27,16 @@ from .models import ( AnalysisGridCell, AnalysisGridObservation, BlockSubdivision, + RemoteSensingClusterBlock, RemoteSensingRun, RemoteSensingSubdivisionResult, + RemoteSensingSubdivisionOption, SoilLocation, ) from farm_data.models import SensorData from .data_driven_subdivision import DEFAULT_CLUSTER_FEATURES +from .data_driven_subdivision import activate_subdivision_option from .serializers import ( BlockSubdivisionSerializer, NdviHealthRequestSerializer, @@ -42,12 +46,25 @@ from .serializers import ( RemoteSensingFarmRequestSerializer, RemoteSensingRunSerializer, RemoteSensingRunStatusResponseSerializer, + RemoteSensingClusterBlockLiveRequestSerializer, + RemoteSensingClusterBlockLiveResponseSerializer, + RemoteSensingClusterBlockSerializer, + RemoteSensingSubdivisionOptionActivateResponseSerializer, + RemoteSensingSubdivisionOptionActivateSerializer, + RemoteSensingSubdivisionOptionListResponseSerializer, + RemoteSensingSubdivisionOptionSerializer, RemoteSensingSummarySerializer, RemoteSensingSubdivisionResultSerializer, SoilDataRequestSerializer, SoilLocationResponseSerializer, ) from .tasks import run_remote_sensing_analysis_task +from .openeo_service import ( + OpenEOAuthenticationError, + OpenEOExecutionError, + OpenEOServiceError, + compute_remote_sensing_metrics, +) MAX_REMOTE_SENSING_PAGE_SIZE = 200 REMOTE_SENSING_RUN_STAGE_ORDER = ( @@ -119,6 +136,18 @@ RemoteSensingRunStatusEnvelopeSerializer = build_envelope_serializer( "RemoteSensingRunStatusEnvelopeSerializer", RemoteSensingRunStatusResponseSerializer, ) +RemoteSensingClusterBlockLiveEnvelopeSerializer = build_envelope_serializer( + "RemoteSensingClusterBlockLiveEnvelopeSerializer", + RemoteSensingClusterBlockLiveResponseSerializer, +) +RemoteSensingSubdivisionOptionListEnvelopeSerializer = build_envelope_serializer( + "RemoteSensingSubdivisionOptionListEnvelopeSerializer", + RemoteSensingSubdivisionOptionListResponseSerializer, +) +RemoteSensingSubdivisionOptionActivateEnvelopeSerializer = build_envelope_serializer( + "RemoteSensingSubdivisionOptionActivateEnvelopeSerializer", + RemoteSensingSubdivisionOptionActivateResponseSerializer, +) class SoilDataView(APIView): """ ثبت مختصات گوشه‌های مزرعه و بلوک‌های تعریف‌شده توسط کشاورز. @@ -679,6 +708,256 @@ class RemoteSensingRunStatusView(APIView): ) +class RemoteSensingClusterBlockLiveView(APIView): + @extend_schema( + tags=["Location Data"], + summary="دریافت زنده remote sensing برای زیر‌بلاک KMeans", + description="با دریافت UUID زیر‌بلاک ساخته‌شده توسط KMeans، هندسه همان زیر‌بلاک از دیتابیس خوانده می‌شود و داده تازه ماهواره‌ای از openEO برگردانده می‌شود.", + parameters=[ + OpenApiParameter( + name="cluster_uuid", + type={"type": "string", "format": "uuid"}, + location=OpenApiParameter.PATH, + required=True, + description="شناسه UUID زیر‌بلاک KMeans.", + ), + OpenApiParameter( + name="temporal_start", + type={"type": "string", "format": "date"}, + location=OpenApiParameter.QUERY, + required=False, + description="شروع بازه سفارشی. اگر ست شود، temporal_end هم باید ارسال شود.", + ), + OpenApiParameter( + name="temporal_end", + type={"type": "string", "format": "date"}, + location=OpenApiParameter.QUERY, + required=False, + description="پایان بازه سفارشی. اگر ست شود، temporal_start هم باید ارسال شود.", + ), + OpenApiParameter( + name="days", + type=int, + location=OpenApiParameter.QUERY, + required=False, + default=30, + description="اگر بازه سفارشی ارسال نشود، از yesterday backfill با این تعداد روز استفاده می‌شود.", + ), + ], + responses={ + 200: build_response( + RemoteSensingClusterBlockLiveEnvelopeSerializer, + "داده زنده openEO برای زیر‌بلاک KMeans بازگردانده شد.", + ), + 400: build_response( + SoilErrorResponseSerializer, + "پارامترهای ورودی یا هندسه زیر‌بلاک نامعتبر است.", + ), + 404: build_response( + SoilErrorResponseSerializer, + "زیر‌بلاک KMeans پیدا نشد.", + ), + 502: build_response( + SoilErrorResponseSerializer, + "خواندن داده از openEO ناموفق بود.", + ), + }, + ) + def get(self, request, cluster_uuid): + serializer = RemoteSensingClusterBlockLiveRequestSerializer(data=request.query_params) + if not serializer.is_valid(): + return Response( + {"code": 400, "msg": "داده نامعتبر.", "data": serializer.errors}, + status=status.HTTP_400_BAD_REQUEST, + ) + + cluster_block = ( + RemoteSensingClusterBlock.objects.select_related("soil_location", "block_subdivision", "result") + .filter(uuid=cluster_uuid) + .first() + ) + if cluster_block is None: + return Response( + {"code": 404, "msg": "زیر‌بلاک KMeans پیدا نشد.", "data": None}, + status=status.HTTP_404_NOT_FOUND, + ) + + geometry = _resolve_cluster_block_geometry(cluster_block) + if not geometry: + return Response( + { + "code": 400, + "msg": "هندسه زیر‌بلاک KMeans نامعتبر است.", + "data": {"cluster_uuid": [str(cluster_block.uuid)]}, + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + temporal_start, temporal_end = _resolve_live_remote_sensing_window(serializer.validated_data) + virtual_cell = _build_virtual_cluster_block_cell(cluster_block=cluster_block, geometry=geometry) + try: + remote_payload = compute_remote_sensing_metrics( + [virtual_cell], + temporal_start=temporal_start, + temporal_end=temporal_end, + selected_features=list(DEFAULT_CLUSTER_FEATURES), + ) + except (OpenEOAuthenticationError, OpenEOExecutionError, OpenEOServiceError) as exc: + return Response( + {"code": 502, "msg": "خواندن داده از openEO ناموفق بود.", "data": {"detail": str(exc)}}, + status=status.HTTP_502_BAD_GATEWAY, + ) + + metrics = dict(remote_payload.get("results", {}).get(virtual_cell.cell_code, {}) or {}) + response_payload = { + "status": "success", + "source": "openeo", + "cluster_block": RemoteSensingClusterBlockSerializer(cluster_block).data, + "temporal_extent": { + "start_date": temporal_start.isoformat(), + "end_date": temporal_end.isoformat(), + }, + "selected_features": list(DEFAULT_CLUSTER_FEATURES), + "summary": { + "cell_count": int(cluster_block.cell_count or 0), + "ndvi_mean": _round_or_none(metrics.get("ndvi")), + "ndwi_mean": _round_or_none(metrics.get("ndwi")), + "soil_vv_db_mean": _round_or_none(metrics.get("soil_vv_db")), + }, + "metrics": { + "ndvi": _round_or_none(metrics.get("ndvi")), + "ndwi": _round_or_none(metrics.get("ndwi")), + "soil_vv": _round_or_none(metrics.get("soil_vv")), + "soil_vv_db": _round_or_none(metrics.get("soil_vv_db")), + }, + "metadata": { + **dict(remote_payload.get("metadata") or {}), + "requested_cluster_uuid": str(cluster_block.uuid), + "block_code": cluster_block.block_code, + "sub_block_code": cluster_block.sub_block_code, + }, + } + return Response( + {"code": 200, "msg": "success", "data": response_payload}, + status=status.HTTP_200_OK, + ) + + +class RemoteSensingSubdivisionOptionListView(APIView): + @extend_schema( + tags=["Location Data"], + summary="فهرست همه گزینه‌های K ذخیره‌شده برای یک subdivision result", + description="همه ترکیب‌های K که برای این run/result محاسبه و ذخیره شده‌اند را برمی‌گرداند و مشخص می‌کند کدام‌یک active و کدام‌یک recommended است.", + responses={ + 200: build_response( + RemoteSensingSubdivisionOptionListEnvelopeSerializer, + "فهرست گزینه‌های K بازگردانده شد.", + ), + 404: build_response( + SoilErrorResponseSerializer, + "subdivision result موردنظر پیدا نشد.", + ), + }, + ) + def get(self, request, result_id): + result = ( + RemoteSensingSubdivisionResult.objects.filter(pk=result_id) + .prefetch_related("options__cluster_blocks") + .first() + ) + if result is None: + return Response( + {"code": 404, "msg": "subdivision result پیدا نشد.", "data": None}, + status=status.HTTP_404_NOT_FOUND, + ) + + options = list(result.options.all().order_by("requested_k")) + response_payload = { + "result_id": result.id, + "active_requested_k": next((option.requested_k for option in options if option.is_active), None), + "recommended_requested_k": next((option.requested_k for option in options if option.is_recommended), None), + "options": RemoteSensingSubdivisionOptionSerializer(options, many=True).data, + } + return Response( + {"code": 200, "msg": "success", "data": response_payload}, + status=status.HTTP_200_OK, + ) + + +class RemoteSensingSubdivisionOptionActivateView(APIView): + @extend_schema( + tags=["Location Data"], + summary="فعال‌سازی یک K ذخیره‌شده برای subdivision result", + description="کاربر می‌تواند یکی از Kهای از قبل محاسبه‌شده و ذخیره‌شده را انتخاب کند تا active شود و خروجی اصلی subdivision بر همان مبنا sync شود.", + request=RemoteSensingSubdivisionOptionActivateSerializer, + responses={ + 200: build_response( + RemoteSensingSubdivisionOptionActivateEnvelopeSerializer, + "K انتخابی فعال شد.", + ), + 400: build_response( + SoilErrorResponseSerializer, + "درخواست نامعتبر است یا K انتخابی موجود نیست.", + ), + 404: build_response( + SoilErrorResponseSerializer, + "subdivision result موردنظر پیدا نشد.", + ), + }, + ) + def post(self, request, result_id): + serializer = RemoteSensingSubdivisionOptionActivateSerializer(data=request.data) + if not serializer.is_valid(): + return Response( + {"code": 400, "msg": "داده نامعتبر.", "data": serializer.errors}, + status=status.HTTP_400_BAD_REQUEST, + ) + + result = ( + RemoteSensingSubdivisionResult.objects.filter(pk=result_id) + .select_related("soil_location", "block_subdivision") + .prefetch_related("options__cluster_blocks", "options__assignments__cell") + .first() + ) + if result is None: + return Response( + {"code": 404, "msg": "subdivision result پیدا نشد.", "data": None}, + status=status.HTTP_404_NOT_FOUND, + ) + + requested_k = serializer.validated_data["requested_k"] + option = next( + (candidate for candidate in result.options.all() if candidate.requested_k == requested_k), + None, + ) + if option is None: + return Response( + { + "code": 400, + "msg": "K انتخابی برای این subdivision result موجود نیست.", + "data": {"requested_k": [requested_k]}, + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + activate_subdivision_option(option=option, selection_source="user") + result.refresh_from_db() + result = ( + RemoteSensingSubdivisionResult.objects.filter(pk=result.pk) + .prefetch_related("assignments__cell", "cluster_blocks", "options__cluster_blocks") + .first() + ) + response_payload = { + "result_id": result.id, + "activated_requested_k": requested_k, + "subdivision_result": RemoteSensingSubdivisionResultSerializer(result).data, + } + return Response( + {"code": 200, "msg": "success", "data": response_payload}, + status=status.HTTP_200_OK, + ) + + def _build_remote_sensing_run_status_payload(run: RemoteSensingRun, *, page: int, page_size: int) -> dict: run_data = RemoteSensingRunSerializer(run).data task_id = (run.metadata or {}).get("task_id") @@ -1069,6 +1348,62 @@ def _resolve_chunk_size_for_location(location: SoilLocation, block_code: str) -> return 900 +def _resolve_live_remote_sensing_window(payload: dict[str, Any]): + temporal_start = payload.get("temporal_start") + temporal_end = payload.get("temporal_end") + if temporal_start and temporal_end: + return temporal_start, temporal_end + days = int(payload.get("days") or 30) + end_date = timezone.localdate() - timedelta(days=1) + start_date = end_date - timedelta(days=days - 1) + return start_date, end_date + + +def _resolve_cluster_block_geometry(cluster_block: RemoteSensingClusterBlock) -> dict[str, Any]: + geometry = dict(cluster_block.geometry or {}) + if geometry.get("type") and geometry.get("coordinates"): + return geometry + + cell_codes = list(cluster_block.cell_codes or []) + if not cell_codes: + return {} + cell_geometries = list( + AnalysisGridCell.objects.filter( + soil_location=cluster_block.soil_location, + cell_code__in=cell_codes, + ) + .order_by("cell_code") + .values_list("geometry", flat=True) + ) + polygon_coordinates: list[list[list[list[float]]]] = [] + for cell_geometry in cell_geometries: + cell_geometry = dict(cell_geometry or {}) + coordinates = cell_geometry.get("coordinates") or [] + if cell_geometry.get("type") == "Polygon" and coordinates: + polygon_coordinates.append(coordinates) + elif cell_geometry.get("type") == "MultiPolygon" and coordinates: + polygon_coordinates.extend(coordinates) + if not polygon_coordinates: + return {} + if len(polygon_coordinates) == 1: + return {"type": "Polygon", "coordinates": polygon_coordinates[0]} + return {"type": "MultiPolygon", "coordinates": polygon_coordinates} + + +def _build_virtual_cluster_block_cell( + *, + cluster_block: RemoteSensingClusterBlock, + geometry: dict[str, Any], +): + return SimpleNamespace( + cell_code=f"cluster-{cluster_block.uuid}", + block_code=cluster_block.block_code, + soil_location_id=cluster_block.soil_location_id, + chunk_size_sqm=cluster_block.chunk_size_sqm, + geometry=geometry, + ) + + def _get_remote_sensing_observations(*, location, block_code: str, start_date, end_date): queryset = ( AnalysisGridObservation.objects.select_related("cell", "run") @@ -1104,7 +1439,7 @@ def _get_remote_sensing_subdivision_result(*, location, block_code: str, start_d temporal_end=end_date, ) .select_related("run") - .prefetch_related("assignments__cell") + .prefetch_related("assignments__cell", "cluster_blocks") .order_by("-created_at", "-id") .first() ) diff --git a/logs/app.log.2026-04-08 b/logs/app.log.2026-04-08 deleted file mode 100644 index 9cd09d4..0000000 --- a/logs/app.log.2026-04-08 +++ /dev/null @@ -1,3 +0,0 @@ -2026-04-08 16:22:28,505 [INFO] django.utils.autoreload: Watching for file changes with StatReloader -2026-04-08 16:23:17,551 [INFO] django.utils.autoreload: Watching for file changes with StatReloader -2026-04-08 16:25:19,939 [INFO] django.utils.autoreload: Watching for file changes with StatReloader