This commit is contained in:
2026-05-09 16:55:06 +03:30
parent 1679825ae2
commit cead7dafe2
51 changed files with 7514 additions and 1221 deletions
+9
View File
@@ -0,0 +1,9 @@
# `location_data` خیلی خلاصه
- ورودی این اپ، مختصات گوشه‌های کل زمین و boundary هر بلوکِ تعریف‌شده توسط کشاورز است.
- هر بلوک جداگانه به grid های `30×30` متر تبدیل می‌شود و در `AnalysisGridCell` ذخیره می‌شود.
- برای همه grid های همان بلوک، داده ماهواره‌ای یک بازه زمانی از `openEO` گرفته می‌شود و میانگین همان بازه به عنوان وضعیت نهایی هر grid در `AnalysisGridObservation` ذخیره می‌شود.
- feature های اصلی فعلی: `ndvi`, `ndwi`, `lst_c`, `soil_vv`, `soil_vv_db`, `dem_m`, `slope_deg`.
- بعد برای هر بلوک، روی feature های grid ها `KMeans` اجرا می‌شود؛ برای هر `K` مقدار `SSE / Inertia` ذخیره می‌شود و نمودار `K-SSE` هم ساخته می‌شود.
- نقطه elbow همان تعداد مناسب زیر‌بلوک‌ها است و نتیجه در `RemoteSensingSubdivisionResult` و خود `BlockSubdivision` ذخیره می‌شود.
- جریان قدیمی depth-based soil data و `soil_adapters.py` دیگر در workflow این اپ جایی ندارد.
+122 -10
View File
@@ -1,11 +1,29 @@
from django.contrib import admin
from .models import SoilDepthData, SoilLocation
from .models import (
AnalysisGridCell,
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingClusterAssignment,
RemoteSensingRun,
RemoteSensingSubdivisionResult,
SoilLocation,
)
class SoilDepthDataInline(admin.TabularInline):
model = SoilDepthData
class BlockSubdivisionInline(admin.TabularInline):
model = BlockSubdivision
extra = 0
readonly_fields = ("depth_label", "bdod", "cec", "cfvo", "clay", "nitrogen", "ocd", "ocs", "phh2o", "sand", "silt", "soc", "wv0010", "wv0033", "wv1500")
readonly_fields = (
"block_code",
"chunk_size_sqm",
"grid_point_count",
"centroid_count",
"status",
"created_at",
"updated_at",
)
fields = readonly_fields
show_change_link = True
@admin.register(SoilLocation)
@@ -14,11 +32,105 @@ class SoilLocationAdmin(admin.ModelAdmin):
list_filter = ("created_at",)
search_fields = ("latitude", "longitude")
readonly_fields = ("created_at", "updated_at")
inlines = [SoilDepthDataInline]
inlines = [BlockSubdivisionInline]
@admin.register(SoilDepthData)
class SoilDepthDataAdmin(admin.ModelAdmin):
list_display = ("id", "soil_location", "depth_label", "bdod", "cec", "phh2o", "clay", "sand", "silt")
list_filter = ("depth_label",)
search_fields = ("soil_location__latitude", "soil_location__longitude")
@admin.register(BlockSubdivision)
class BlockSubdivisionAdmin(admin.ModelAdmin):
list_display = (
"id",
"soil_location",
"block_code",
"chunk_size_sqm",
"grid_point_count",
"centroid_count",
"status",
"updated_at",
)
list_filter = ("status", "chunk_size_sqm", "created_at")
search_fields = ("block_code", "soil_location__latitude", "soil_location__longitude")
readonly_fields = ("created_at", "updated_at")
@admin.register(RemoteSensingRun)
class RemoteSensingRunAdmin(admin.ModelAdmin):
list_display = (
"id",
"soil_location",
"block_code",
"provider",
"chunk_size_sqm",
"status",
"temporal_start",
"temporal_end",
"created_at",
)
list_filter = ("provider", "status", "chunk_size_sqm", "created_at")
search_fields = ("block_code", "soil_location__latitude", "soil_location__longitude")
readonly_fields = ("created_at", "updated_at")
@admin.register(AnalysisGridCell)
class AnalysisGridCellAdmin(admin.ModelAdmin):
list_display = (
"id",
"cell_code",
"soil_location",
"block_code",
"chunk_size_sqm",
"centroid_lat",
"centroid_lon",
"created_at",
)
list_filter = ("chunk_size_sqm", "created_at")
search_fields = ("cell_code", "block_code", "soil_location__latitude", "soil_location__longitude")
readonly_fields = ("created_at", "updated_at")
@admin.register(AnalysisGridObservation)
class AnalysisGridObservationAdmin(admin.ModelAdmin):
list_display = (
"id",
"cell",
"temporal_start",
"temporal_end",
"ndvi",
"ndwi",
"lst_c",
"created_at",
)
list_filter = ("temporal_start", "temporal_end", "created_at")
search_fields = ("cell__cell_code", "cell__block_code")
readonly_fields = ("created_at", "updated_at")
@admin.register(RemoteSensingSubdivisionResult)
class RemoteSensingSubdivisionResultAdmin(admin.ModelAdmin):
list_display = (
"id",
"soil_location",
"block_code",
"cluster_count",
"chunk_size_sqm",
"temporal_start",
"temporal_end",
"created_at",
)
list_filter = ("chunk_size_sqm", "cluster_count", "created_at")
search_fields = ("block_code", "soil_location__latitude", "soil_location__longitude")
readonly_fields = ("created_at", "updated_at")
@admin.register(RemoteSensingClusterAssignment)
class RemoteSensingClusterAssignmentAdmin(admin.ModelAdmin):
list_display = (
"id",
"result",
"cell",
"cluster_label",
"created_at",
)
list_filter = ("cluster_label", "created_at")
search_fields = ("cell__cell_code", "result__block_code")
readonly_fields = ("created_at", "updated_at")
+1 -22
View File
@@ -1,13 +1,12 @@
from functools import cached_property
from django.apps import AppConfig
from django.conf import settings
class SoilDataConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "location_data"
verbose_name = "Soil Data (SoilGrids)"
verbose_name = "Location Data (Remote Sensing)"
@cached_property
def ndvi_health_service(self):
@@ -15,25 +14,5 @@ class SoilDataConfig(AppConfig):
return NdviHealthService()
@cached_property
def soil_data_adapter(self):
from .soil_adapters import MockSoilDataAdapter, SoilGridsAdapter
provider = getattr(settings, "SOIL_DATA_PROVIDER", "soilgrids")
if provider == "soilgrids":
return SoilGridsAdapter(
timeout=getattr(settings, "SOILGRIDS_TIMEOUT_SECONDS", 60)
)
if provider == "mock":
if not (getattr(settings, "DEBUG", False) or getattr(settings, "DEVELOP", False)):
raise RuntimeError("Mock soil provider is disabled outside dev/test environments.")
return MockSoilDataAdapter(
delay_seconds=getattr(settings, "SOIL_MOCK_DELAY_SECONDS", 0.8)
)
raise ValueError(f"Unsupported soil data provider: {provider}")
def get_ndvi_health_service(self):
return self.ndvi_health_service
def get_soil_data_adapter(self):
return self.soil_data_adapter
+401
View File
@@ -0,0 +1,401 @@
from __future__ import annotations
from dataclasses import dataclass
from decimal import Decimal, ROUND_HALF_UP
from io import BytesIO
import math
from django.conf import settings
from django.core.files.base import ContentFile
EARTH_RADIUS_M = 6371008.8
COORD_PRECISION = Decimal("0.000001")
MAX_K = 10
RANDOM_STATE = 42
@dataclass(frozen=True)
class GeoPoint:
lat: float
lon: float
def create_or_get_block_subdivision(
location,
block_code: str,
boundary: dict | list,
*,
chunk_size_sqm: int | None = None,
):
"""
اگر subdivision این بلوک قبلاً ساخته شده باشد همان را برمی‌گرداند؛
در غیر این صورت الگوریتم grid + KMeans را اجرا و ذخیره می‌کند.
"""
from .models import BlockSubdivision
existing = BlockSubdivision.objects.filter(
soil_location=location,
block_code=block_code,
).first()
if existing is not None:
return existing, False
payload = build_block_subdivision_payload(
boundary=boundary,
block_code=block_code,
chunk_size_sqm=chunk_size_sqm,
)
subdivision = BlockSubdivision.objects.create(
soil_location=location,
block_code=block_code,
source_boundary=payload["source_boundary"],
chunk_size_sqm=payload["chunk_size_sqm"],
grid_points=payload["grid_points"],
centroid_points=payload["centroid_points"],
grid_point_count=payload["grid_point_count"],
centroid_count=payload["centroid_count"],
status="created",
metadata=payload["metadata"],
)
plot_content = render_elbow_plot(
inertia_curve=payload["metadata"].get("inertia_curve", []),
optimal_k=payload["metadata"].get("optimal_k", 0),
block_code=block_code,
)
if plot_content is not None:
subdivision.elbow_plot.save(
f"{location.pk}_{block_code}_elbow.png",
plot_content,
save=False,
)
subdivision.save(update_fields=["elbow_plot", "updated_at"])
sync_block_layout_with_subdivision(location, subdivision)
return subdivision, True
def build_block_subdivision_payload(
boundary: dict | list,
block_code: str = "block-1",
chunk_size_sqm: int | None = None,
) -> dict:
"""
مرز یک بلوک را گرفته و ابتدا شبکه نقاط را می‌سازد، سپس با KMeans
تعداد بهینه خوشه‌ها را از elbow point پیدا می‌کند و centroidها را برمی‌گرداند.
"""
chunk_size = int(chunk_size_sqm or getattr(settings, "SUBDIVISION_CHUNK_SQM", 900) or 900)
if chunk_size <= 0:
raise ValueError("chunk_size_sqm باید بزرگ‌تر از صفر باشد.")
polygon = extract_polygon(boundary)
if len(polygon) < 3:
raise ValueError("مرز بلوک باید حداقل سه نقطه معتبر داشته باشد.")
projected_polygon = project_polygon_to_local_meters(polygon)
area_sqm = abs(polygon_area(projected_polygon))
grid_points, grid_vectors = generate_grid_points(
polygon=polygon,
projected_polygon=projected_polygon,
chunk_size_sqm=chunk_size,
)
clustering_result = cluster_grid_points(grid_vectors, polygon)
return {
"block_code": block_code,
"source_boundary": boundary if isinstance(boundary, dict) else {"points": boundary},
"chunk_size_sqm": chunk_size,
"grid_points": grid_points,
"centroid_points": clustering_result["centroid_points"],
"grid_point_count": len(grid_points),
"centroid_count": len(clustering_result["centroid_points"]),
"metadata": {
"estimated_area_sqm": round(area_sqm, 2),
"optimal_k": clustering_result["optimal_k"],
"inertia_curve": clustering_result["inertia_curve"],
},
}
def cluster_grid_points(grid_vectors: list[tuple[float, float]], polygon: list[GeoPoint]) -> dict:
if not grid_vectors:
return {
"optimal_k": 0,
"inertia_curve": [],
"centroid_points": [],
}
if len(grid_vectors) == 1:
lat, lon = unproject_point(grid_vectors[0][0], grid_vectors[0][1], polygon)
return {
"optimal_k": 1,
"inertia_curve": [{"k": 1, "sse": 0.0}],
"centroid_points": [
{
"sub_block_code": "sub-block-1",
"centroid_lat": quantize_coordinate(lat),
"centroid_lon": quantize_coordinate(lon),
}
],
}
try:
from sklearn.cluster import KMeans
except ImportError as exc: # pragma: no cover - runtime dependency guard
raise ImportError("scikit-learn برای اجرای subdivision لازم است.") from exc
max_k = min(MAX_K, len(grid_vectors))
inertia_curve = []
trained_models = {}
for k in range(1, max_k + 1):
model = KMeans(
n_clusters=k,
n_init=10,
random_state=RANDOM_STATE,
)
model.fit(grid_vectors)
trained_models[k] = model
inertia_curve.append({"k": k, "sse": round(float(model.inertia_), 6)})
optimal_k = detect_elbow_point(inertia_curve)
final_model = trained_models[optimal_k]
centroid_points = []
for index, center in enumerate(final_model.cluster_centers_, start=1):
lat, lon = unproject_point(center[0], center[1], polygon)
centroid_points.append(
{
"sub_block_code": f"sub-block-{index}",
"centroid_lat": quantize_coordinate(lat),
"centroid_lon": quantize_coordinate(lon),
}
)
return {
"optimal_k": optimal_k,
"inertia_curve": inertia_curve,
"centroid_points": centroid_points,
}
def detect_elbow_point(inertia_curve: list[dict]) -> int:
if not inertia_curve:
return 0
if len(inertia_curve) <= 2:
return inertia_curve[-1]["k"] if len(inertia_curve) == 2 else inertia_curve[0]["k"]
sses = [item["sse"] for item in inertia_curve]
ks = [item["k"] for item in inertia_curve]
slopes = [sses[index] - sses[index + 1] for index in range(len(sses) - 1)]
best_k = ks[0]
best_change = float("-inf")
for index in range(len(slopes) - 1):
change = slopes[index] - slopes[index + 1]
candidate_k = ks[index + 1]
if change > best_change:
best_change = change
best_k = candidate_k
return best_k
def render_elbow_plot(
inertia_curve: list[dict],
optimal_k: int,
block_code: str,
) -> ContentFile | None:
if not inertia_curve:
return None
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError as exc: # pragma: no cover - runtime dependency guard
raise ImportError("matplotlib برای ذخیره نمودار elbow لازم است.") from exc
ks = [item["k"] for item in inertia_curve]
sses = [item["sse"] for item in inertia_curve]
buffer = BytesIO()
fig, ax = plt.subplots(figsize=(8, 5))
try:
ax.plot(ks, sses, marker="o", linewidth=2, color="#2f6fed")
if optimal_k in ks:
elbow_index = ks.index(optimal_k)
ax.scatter(
[ks[elbow_index]],
[sses[elbow_index]],
color="#d62828",
s=90,
zorder=3,
label=f"Elbow K={optimal_k}",
)
ax.legend()
ax.set_title(f"Elbow Plot - {block_code}")
ax.set_xlabel("K")
ax.set_ylabel("SSE / Inertia")
ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.6)
fig.tight_layout()
fig.savefig(buffer, format="png", dpi=150)
buffer.seek(0)
return ContentFile(buffer.getvalue())
finally:
buffer.close()
plt.close(fig)
def sync_block_layout_with_subdivision(location, subdivision) -> None:
layout = location.block_layout or {}
blocks = list(layout.get("blocks") or [])
target_block = None
for block in blocks:
if block.get("block_code") == subdivision.block_code:
target_block = block
break
if target_block is None:
target_block = {
"block_code": subdivision.block_code,
"order": len(blocks) + 1,
"source": "input",
"needs_subdivision": None,
"sub_blocks": [],
}
blocks.append(target_block)
target_block["needs_subdivision"] = subdivision.centroid_count > 1
target_block["sub_blocks"] = list(subdivision.centroid_points or [])
target_block["subdivision_summary"] = {
"chunk_size_sqm": subdivision.chunk_size_sqm,
"grid_point_count": subdivision.grid_point_count,
"centroid_count": subdivision.centroid_count,
"optimal_k": (subdivision.metadata or {}).get("optimal_k", subdivision.centroid_count),
}
layout["blocks"] = blocks
layout["algorithm_status"] = "completed"
location.block_layout = layout
location.save(update_fields=["block_layout", "updated_at"])
def generate_grid_points(
polygon: list[GeoPoint],
projected_polygon: list[tuple[float, float]],
chunk_size_sqm: int,
) -> tuple[list[dict], list[tuple[float, float]]]:
step_m = math.sqrt(chunk_size_sqm)
min_x, max_x, min_y, max_y = bounds(projected_polygon)
grid_points: list[dict] = []
grid_vectors: list[tuple[float, float]] = []
y = min_y + (step_m / 2.0)
point_index = 0
while y <= max_y:
x = min_x + (step_m / 2.0)
while x <= max_x:
if point_in_polygon((x, y), projected_polygon):
lat, lon = unproject_point(x, y, polygon)
point_index += 1
grid_vectors.append((x, y))
grid_points.append(
{
"point_code": f"pt-{point_index}",
"lat": quantize_coordinate(lat),
"lon": quantize_coordinate(lon),
}
)
x += step_m
y += step_m
return grid_points, grid_vectors
def extract_polygon(boundary: dict | list) -> list[GeoPoint]:
if isinstance(boundary, dict):
if boundary.get("type") == "Polygon":
coordinates = boundary.get("coordinates") or []
if coordinates and isinstance(coordinates[0], list):
points = coordinates[0]
else:
points = []
else:
points = boundary.get("corners") or []
elif isinstance(boundary, list):
points = boundary
else:
points = []
polygon: list[GeoPoint] = []
for point in points:
lat = lon = None
if isinstance(point, dict):
lat = point.get("lat", point.get("latitude"))
lon = point.get("lon", point.get("longitude"))
elif isinstance(point, (list, tuple)) and len(point) >= 2:
lon, lat = point[0], point[1]
if lat is None or lon is None:
continue
polygon.append(GeoPoint(lat=float(lat), lon=float(lon)))
if len(polygon) > 1 and polygon[0] == polygon[-1]:
polygon = polygon[:-1]
return polygon
def project_polygon_to_local_meters(polygon: list[GeoPoint]) -> list[tuple[float, float]]:
origin = polygon[0]
lat0 = math.radians(origin.lat)
lon0 = math.radians(origin.lon)
cos_lat0 = math.cos(lat0)
projected = []
for point in polygon:
lat = math.radians(point.lat)
lon = math.radians(point.lon)
x = (lon - lon0) * cos_lat0 * EARTH_RADIUS_M
y = (lat - lat0) * EARTH_RADIUS_M
projected.append((x, y))
return projected
def unproject_point(x: float, y: float, polygon: list[GeoPoint]) -> tuple[float, float]:
origin = polygon[0]
lat0 = math.radians(origin.lat)
lon0 = math.radians(origin.lon)
cos_lat0 = math.cos(lat0)
lat = math.degrees((y / EARTH_RADIUS_M) + lat0)
lon = math.degrees((x / (EARTH_RADIUS_M * cos_lat0)) + lon0)
return lat, lon
def polygon_area(points: list[tuple[float, float]]) -> float:
area = 0.0
closed = points + [points[0]]
for index in range(len(points)):
x1, y1 = closed[index]
x2, y2 = closed[index + 1]
area += (x1 * y2) - (x2 * y1)
return area / 2.0
def bounds(points: list[tuple[float, float]]) -> tuple[float, float, float, float]:
xs = [point[0] for point in points]
ys = [point[1] for point in points]
return min(xs), max(xs), min(ys), max(ys)
def point_in_polygon(point: tuple[float, float], polygon: list[tuple[float, float]]) -> bool:
x, y = point
inside = False
j = len(polygon) - 1
for i in range(len(polygon)):
xi, yi = polygon[i]
xj, yj = polygon[j]
intersects = ((yi > y) != (yj > y)) and (
x < ((xj - xi) * (y - yi) / ((yj - yi) or 1e-12)) + xi
)
if intersects:
inside = not inside
j = i
return inside
def quantize_coordinate(value: float) -> float:
return float(Decimal(str(value)).quantize(COORD_PRECISION, rounding=ROUND_HALF_UP))
+489
View File
@@ -0,0 +1,489 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from django.db import transaction
from .block_subdivision import detect_elbow_point, render_elbow_plot
from .models import (
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingClusterAssignment,
RemoteSensingRun,
RemoteSensingSubdivisionResult,
SoilLocation,
)
DEFAULT_CLUSTER_FEATURES = [
"ndvi",
"ndwi",
"lst_c",
"soil_vv_db",
"dem_m",
"slope_deg",
]
SUPPORTED_CLUSTER_FEATURES = tuple(DEFAULT_CLUSTER_FEATURES)
DEFAULT_RANDOM_STATE = 42
DEFAULT_MAX_K = 10
class DataDrivenSubdivisionError(Exception):
"""Raised when remote-sensing-driven subdivision can not be computed."""
@dataclass
class ClusteringDataset:
observations: list[AnalysisGridObservation]
selected_features: list[str]
raw_feature_rows: list[list[float | None]]
raw_feature_maps: list[dict[str, float | None]]
skipped_cell_codes: list[str]
used_cell_codes: list[str]
imputed_matrix: list[list[float]]
scaled_matrix: list[list[float]]
imputer_statistics: dict[str, float | None]
scaler_means: dict[str, float]
scaler_scales: dict[str, float]
missing_value_counts: dict[str, int]
skipped_reasons: dict[str, list[str]]
def create_remote_sensing_subdivision_result(
*,
location: SoilLocation,
run: RemoteSensingRun,
observations: list[AnalysisGridObservation],
block_subdivision: BlockSubdivision | None = None,
block_code: str = "",
selected_features: list[str] | None = None,
explicit_k: int | None = None,
max_k: int = DEFAULT_MAX_K,
random_state: int = DEFAULT_RANDOM_STATE,
) -> RemoteSensingSubdivisionResult:
"""
Build a data-driven subdivision result from stored remote sensing observations.
KMeans is applied on actual per-cell feature vectors, not geometric points.
"""
dataset = build_clustering_dataset(
observations=observations,
selected_features=selected_features,
)
if not dataset.observations:
raise DataDrivenSubdivisionError("هیچ observation قابل استفاده‌ای برای خوشه‌بندی باقی نماند.")
optimal_k, inertia_curve = choose_cluster_count(
scaled_matrix=dataset.scaled_matrix,
explicit_k=explicit_k,
max_k=max_k,
random_state=random_state,
)
cluster_selection_strategy = "explicit_k" if explicit_k is not None else "elbow"
labels = run_kmeans_labels(
scaled_matrix=dataset.scaled_matrix,
cluster_count=optimal_k,
random_state=random_state,
)
cluster_summaries = build_cluster_summaries(
observations=dataset.observations,
labels=labels,
)
with transaction.atomic():
result, _created = RemoteSensingSubdivisionResult.objects.update_or_create(
run=run,
defaults={
"soil_location": location,
"block_subdivision": block_subdivision,
"block_code": block_code,
"chunk_size_sqm": run.chunk_size_sqm,
"temporal_start": run.temporal_start,
"temporal_end": run.temporal_end,
"cluster_count": optimal_k,
"selected_features": dataset.selected_features,
"skipped_cell_codes": dataset.skipped_cell_codes,
"metadata": {
"cell_count": len(observations),
"used_cell_count": len(dataset.observations),
"skipped_cell_count": len(dataset.skipped_cell_codes),
"used_cell_codes": dataset.used_cell_codes,
"skipped_reasons": dataset.skipped_reasons,
"selected_features": dataset.selected_features,
"imputer_strategy": "median",
"imputer_statistics": dataset.imputer_statistics,
"missing_value_counts": dataset.missing_value_counts,
"scaler_means": dataset.scaler_means,
"scaler_scales": dataset.scaler_scales,
"kmeans_params": {
"random_state": random_state,
"explicit_k": explicit_k,
"selected_k": optimal_k,
"max_k": max_k,
"n_init": 10,
"selection_strategy": cluster_selection_strategy,
},
"inertia_curve": inertia_curve,
"cluster_summaries": cluster_summaries,
},
},
)
result.assignments.all().delete()
assignment_rows = []
for index, observation in enumerate(dataset.observations):
assignment_rows.append(
RemoteSensingClusterAssignment(
result=result,
cell=observation.cell,
cluster_label=int(labels[index]),
raw_feature_values=dataset.raw_feature_maps[index],
scaled_feature_values={
feature_name: round(dataset.scaled_matrix[index][feature_index], 6)
for feature_index, feature_name in enumerate(dataset.selected_features)
},
)
)
RemoteSensingClusterAssignment.objects.bulk_create(assignment_rows)
if block_subdivision is not None:
sync_block_subdivision_with_result(
block_subdivision=block_subdivision,
result=result,
observations=observations,
cluster_summaries=cluster_summaries,
)
sync_location_block_layout_with_result(
location=location,
result=result,
cluster_summaries=cluster_summaries,
)
return result
def build_clustering_dataset(
*,
observations: list[AnalysisGridObservation],
selected_features: list[str] | None = None,
) -> ClusteringDataset:
selected_features = list(selected_features or DEFAULT_CLUSTER_FEATURES)
invalid_features = [
feature_name
for feature_name in selected_features
if feature_name not in SUPPORTED_CLUSTER_FEATURES
]
if invalid_features:
raise DataDrivenSubdivisionError(
"ویژگی‌های نامعتبر برای خوشه‌بندی: "
+ ", ".join(sorted(invalid_features))
)
raw_rows: list[list[float | None]] = []
raw_maps: list[dict[str, float | None]] = []
usable_observations: list[AnalysisGridObservation] = []
skipped_cell_codes: list[str] = []
used_cell_codes: list[str] = []
missing_value_counts = {feature_name: 0 for feature_name in selected_features}
skipped_reasons = {"all_features_missing": []}
for observation in observations:
feature_map = {
feature_name: _coerce_float(getattr(observation, feature_name, None))
for feature_name in selected_features
}
for feature_name, value in feature_map.items():
if value is None:
missing_value_counts[feature_name] += 1
if all(value is None for value in feature_map.values()):
skipped_cell_codes.append(observation.cell.cell_code)
skipped_reasons["all_features_missing"].append(observation.cell.cell_code)
continue
usable_observations.append(observation)
used_cell_codes.append(observation.cell.cell_code)
raw_maps.append(feature_map)
raw_rows.append([feature_map[feature_name] for feature_name in selected_features])
if not usable_observations:
return ClusteringDataset(
observations=[],
selected_features=selected_features,
raw_feature_rows=[],
raw_feature_maps=[],
skipped_cell_codes=skipped_cell_codes,
used_cell_codes=[],
imputed_matrix=[],
scaled_matrix=[],
imputer_statistics={feature_name: None for feature_name in selected_features},
scaler_means={feature_name: 0.0 for feature_name in selected_features},
scaler_scales={feature_name: 1.0 for feature_name in selected_features},
missing_value_counts=missing_value_counts,
skipped_reasons=skipped_reasons,
)
try:
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
except ImportError as exc: # pragma: no cover - runtime dependency guard
raise DataDrivenSubdivisionError(
"scikit-learn و numpy برای خوشه‌بندی داده‌محور لازم هستند."
) from exc
raw_matrix = np.array(raw_rows, dtype=float)
imputer = SimpleImputer(strategy="median")
imputed_matrix = imputer.fit_transform(raw_matrix)
scaler = StandardScaler()
scaled_matrix = scaler.fit_transform(imputed_matrix)
return ClusteringDataset(
observations=usable_observations,
selected_features=selected_features,
raw_feature_rows=raw_rows,
raw_feature_maps=raw_maps,
skipped_cell_codes=skipped_cell_codes,
used_cell_codes=used_cell_codes,
imputed_matrix=imputed_matrix.tolist(),
scaled_matrix=scaled_matrix.tolist(),
imputer_statistics={
feature_name: _coerce_float(imputer.statistics_[index])
for index, feature_name in enumerate(selected_features)
},
scaler_means={
feature_name: float(scaler.mean_[index])
for index, feature_name in enumerate(selected_features)
},
scaler_scales={
feature_name: float(scaler.scale_[index] or 1.0)
for index, feature_name in enumerate(selected_features)
},
missing_value_counts=missing_value_counts,
skipped_reasons=skipped_reasons,
)
def choose_cluster_count(
*,
scaled_matrix: list[list[float]],
explicit_k: int | None,
max_k: int,
random_state: int,
) -> tuple[int, list[dict[str, float]]]:
sample_count = len(scaled_matrix)
if sample_count == 0:
raise DataDrivenSubdivisionError("هیچ نمونه‌ای برای خوشه‌بندی وجود ندارد.")
if sample_count == 1:
return 1, [{"k": 1, "sse": 0.0}]
if explicit_k is not None:
if explicit_k <= 0:
raise DataDrivenSubdivisionError("cluster_count باید بزرگ‌تر از صفر باشد.")
return min(explicit_k, sample_count), []
try:
from sklearn.cluster import KMeans
except ImportError as exc: # pragma: no cover
raise DataDrivenSubdivisionError("scikit-learn برای انتخاب تعداد خوشه لازم است.") from exc
max_allowed_k = min(max_k, sample_count)
inertia_curve = []
for k in range(1, max_allowed_k + 1):
model = KMeans(n_clusters=k, n_init=10, random_state=random_state)
model.fit(scaled_matrix)
inertia_curve.append({"k": k, "sse": round(float(model.inertia_), 6)})
return detect_elbow_point(inertia_curve), inertia_curve
def run_kmeans_labels(
*,
scaled_matrix: list[list[float]],
cluster_count: int,
random_state: int,
) -> list[int]:
if cluster_count <= 0:
raise DataDrivenSubdivisionError("cluster_count باید بزرگ‌تر از صفر باشد.")
if len(scaled_matrix) == 1:
return [0]
try:
from sklearn.cluster import KMeans
except ImportError as exc: # pragma: no cover
raise DataDrivenSubdivisionError("scikit-learn برای اجرای KMeans لازم است.") from exc
model = KMeans(n_clusters=cluster_count, n_init=10, random_state=random_state)
return [int(label) for label in model.fit_predict(scaled_matrix)]
def build_cluster_summaries(
*,
observations: list[AnalysisGridObservation],
labels: list[int],
) -> list[dict[str, Any]]:
clusters: dict[int, dict[str, Any]] = {}
for observation, label in zip(observations, labels):
cluster = clusters.setdefault(
int(label),
{
"cluster_label": int(label),
"cell_codes": [],
"centroid_lat_sum": 0.0,
"centroid_lon_sum": 0.0,
"cell_count": 0,
},
)
cluster["cell_codes"].append(observation.cell.cell_code)
cluster["centroid_lat_sum"] += float(observation.cell.centroid_lat)
cluster["centroid_lon_sum"] += float(observation.cell.centroid_lon)
cluster["cell_count"] += 1
summaries = []
for cluster_label in sorted(clusters):
cluster = clusters[cluster_label]
cell_count = cluster["cell_count"] or 1
summaries.append(
{
"cluster_label": cluster_label,
"cell_count": cluster["cell_count"],
"centroid_lat": round(cluster["centroid_lat_sum"] / cell_count, 6),
"centroid_lon": round(cluster["centroid_lon_sum"] / cell_count, 6),
"cell_codes": cluster["cell_codes"],
}
)
return summaries
def sync_location_block_layout_with_result(
*,
location: SoilLocation,
result: RemoteSensingSubdivisionResult,
cluster_summaries: list[dict[str, Any]],
) -> None:
layout = dict(location.block_layout or {})
blocks = list(layout.get("blocks") or [])
target_block = None
for block in blocks:
if block.get("block_code") == result.block_code:
target_block = block
break
if target_block is None:
target_block = {
"block_code": result.block_code,
"order": len(blocks) + 1,
"source": "remote_sensing",
"needs_subdivision": None,
"sub_blocks": [],
}
blocks.append(target_block)
target_block["needs_subdivision"] = result.cluster_count > 1
target_block["sub_blocks"] = [
{
"sub_block_code": f"cluster-{cluster['cluster_label']}",
"cluster_label": cluster["cluster_label"],
"centroid_lat": cluster["centroid_lat"],
"centroid_lon": cluster["centroid_lon"],
"cell_count": cluster["cell_count"],
}
for cluster in cluster_summaries
]
target_block["subdivision_summary"] = {
"type": "data_driven_remote_sensing",
"cluster_count": result.cluster_count,
"selected_features": result.selected_features,
"used_cell_count": result.metadata.get("used_cell_count", 0),
"skipped_cell_count": result.metadata.get("skipped_cell_count", 0),
"run_id": result.run_id,
}
layout["blocks"] = blocks
layout["algorithm_status"] = "completed"
location.block_layout = layout
location.save(update_fields=["block_layout", "updated_at"])
def sync_block_subdivision_with_result(
*,
block_subdivision: BlockSubdivision,
result: RemoteSensingSubdivisionResult,
observations: list[AnalysisGridObservation],
cluster_summaries: list[dict[str, Any]],
) -> None:
metadata = dict(block_subdivision.metadata or {})
metadata["data_driven_subdivision"] = {
"run_id": result.run_id,
"result_id": result.id,
"cluster_count": result.cluster_count,
"selected_features": result.selected_features,
"used_cell_count": result.metadata.get("used_cell_count", 0),
"skipped_cell_count": result.metadata.get("skipped_cell_count", 0),
"temporal_extent": {
"start_date": result.temporal_start.isoformat() if result.temporal_start else None,
"end_date": result.temporal_end.isoformat() if result.temporal_end else None,
},
"inertia_curve": result.metadata.get("inertia_curve", []),
}
block_subdivision.grid_points = [
{
"cell_code": observation.cell.cell_code,
"centroid_lat": round(float(observation.cell.centroid_lat), 6),
"centroid_lon": round(float(observation.cell.centroid_lon), 6),
}
for observation in observations
]
block_subdivision.centroid_points = [
{
"sub_block_code": f"cluster-{cluster['cluster_label']}",
"cluster_label": cluster["cluster_label"],
"centroid_lat": cluster["centroid_lat"],
"centroid_lon": cluster["centroid_lon"],
"cell_count": cluster["cell_count"],
"cell_codes": cluster["cell_codes"],
}
for cluster in cluster_summaries
]
block_subdivision.grid_point_count = len(observations)
block_subdivision.centroid_count = len(cluster_summaries)
block_subdivision.status = "subdivided"
block_subdivision.metadata = metadata
plot_content = render_elbow_plot(
inertia_curve=result.metadata.get("inertia_curve", []),
optimal_k=result.cluster_count,
block_code=result.block_code or block_subdivision.block_code,
)
if plot_content is not None:
block_subdivision.elbow_plot.save(
f"remote-sensing-{result.soil_location_id}-{result.block_code or block_subdivision.block_code}-elbow.png",
plot_content,
save=False,
)
block_subdivision.save(
update_fields=[
"grid_points",
"centroid_points",
"grid_point_count",
"centroid_count",
"status",
"metadata",
"elbow_plot",
"updated_at",
]
)
return
block_subdivision.save(
update_fields=[
"grid_points",
"centroid_points",
"grid_point_count",
"centroid_count",
"status",
"metadata",
"updated_at",
]
)
def _coerce_float(value: Any) -> float | None:
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
+327
View File
@@ -0,0 +1,327 @@
from __future__ import annotations
from decimal import Decimal
import math
from django.conf import settings
from django.db import transaction
from .block_subdivision import (
GeoPoint,
bounds,
extract_polygon,
point_in_polygon,
project_polygon_to_local_meters,
quantize_coordinate,
unproject_point,
)
from .models import AnalysisGridCell, BlockSubdivision, SoilLocation
def create_or_get_analysis_grid_cells(
location: SoilLocation,
*,
boundary: dict | list | None = None,
block_code: str | None = None,
block_subdivision: BlockSubdivision | None = None,
chunk_size_sqm: int | None = None,
) -> dict:
"""
شبکه تحلیل 30x30 متری یا هر chunk size تنظیم‌شده را برای مزرعه/بلوک می‌سازد
و رکوردهای AnalysisGridCell را به‌صورت idempotent ذخیره می‌کند.
"""
normalized_chunk_size = int(
chunk_size_sqm or getattr(settings, "SUBDIVISION_CHUNK_SQM", 900) or 900
)
if normalized_chunk_size <= 0:
raise ValueError("chunk_size_sqm باید بزرگ‌تر از صفر باشد.")
resolved_block_code = str(block_code or getattr(block_subdivision, "block_code", "") or "").strip()
resolved_boundary = _resolve_boundary(
location=location,
boundary=boundary,
block_subdivision=block_subdivision,
)
polygon = extract_polygon(resolved_boundary)
if len(polygon) < 3:
raise ValueError("برای ساخت analysis grid باید حداقل سه نقطه معتبر در boundary وجود داشته باشد.")
existing_qs = AnalysisGridCell.objects.filter(
soil_location=location,
block_code=resolved_block_code,
chunk_size_sqm=normalized_chunk_size,
).order_by("cell_code")
existing_count = existing_qs.count()
if existing_count:
return {
"created_count": 0,
"existing_count": existing_count,
"total_count": existing_count,
"chunk_size_sqm": normalized_chunk_size,
"block_code": resolved_block_code,
"created": False,
}
cell_payloads = build_analysis_grid_payload(
polygon=polygon,
location=location,
block_code=resolved_block_code,
chunk_size_sqm=normalized_chunk_size,
)
created_cells = []
with transaction.atomic():
for payload in cell_payloads:
created_cells.append(
AnalysisGridCell.objects.create(
soil_location=location,
block_subdivision=block_subdivision,
block_code=resolved_block_code,
cell_code=payload["cell_code"],
chunk_size_sqm=normalized_chunk_size,
geometry=payload["geometry"],
centroid_lat=Decimal(str(payload["centroid_lat"])),
centroid_lon=Decimal(str(payload["centroid_lon"])),
)
)
_update_grid_summary_metadata(
location=location,
block_code=resolved_block_code,
chunk_size_sqm=normalized_chunk_size,
total_count=len(created_cells),
block_subdivision=block_subdivision,
)
return {
"created_count": len(created_cells),
"existing_count": 0,
"total_count": len(created_cells),
"chunk_size_sqm": normalized_chunk_size,
"block_code": resolved_block_code,
"created": True,
}
def build_analysis_grid_payload(
*,
polygon: list[GeoPoint],
location: SoilLocation,
block_code: str,
chunk_size_sqm: int,
) -> list[dict]:
projected_polygon = project_polygon_to_local_meters(polygon)
step_m = math.sqrt(chunk_size_sqm)
min_x, max_x, min_y, max_y = bounds(projected_polygon)
payloads: list[dict] = []
row_index = 0
y = min_y
while y < max_y:
col_index = 0
x = min_x
while x < max_x:
cell_polygon = [
(x, y),
(x + step_m, y),
(x + step_m, y + step_m),
(x, y + step_m),
]
if _cell_intersects_polygon(cell_polygon, projected_polygon):
payloads.append(
_build_cell_payload(
location=location,
block_code=block_code,
chunk_size_sqm=chunk_size_sqm,
polygon=polygon,
cell_polygon=cell_polygon,
row_index=row_index,
col_index=col_index,
)
)
col_index += 1
x += step_m
row_index += 1
y += step_m
return payloads
def _build_cell_payload(
*,
location: SoilLocation,
block_code: str,
chunk_size_sqm: int,
polygon: list[GeoPoint],
cell_polygon: list[tuple[float, float]],
row_index: int,
col_index: int,
) -> dict:
closed_polygon = cell_polygon + [cell_polygon[0]]
geometry_coordinates = []
for x, y in closed_polygon:
lat, lon = unproject_point(x, y, polygon)
geometry_coordinates.append(
[quantize_coordinate(lon), quantize_coordinate(lat)]
)
centroid_x = sum(point[0] for point in cell_polygon) / len(cell_polygon)
centroid_y = sum(point[1] for point in cell_polygon) / len(cell_polygon)
centroid_lat, centroid_lon = unproject_point(centroid_x, centroid_y, polygon)
return {
"cell_code": build_analysis_cell_code(
location_id=location.id,
block_code=block_code,
chunk_size_sqm=chunk_size_sqm,
row_index=row_index,
col_index=col_index,
),
"geometry": {
"type": "Polygon",
"coordinates": [geometry_coordinates],
},
"centroid_lat": quantize_coordinate(centroid_lat),
"centroid_lon": quantize_coordinate(centroid_lon),
}
def build_analysis_cell_code(
*,
location_id: int | None,
block_code: str,
chunk_size_sqm: int,
row_index: int,
col_index: int,
) -> str:
block_segment = block_code or "farm"
location_segment = location_id if location_id is not None else "new"
return (
f"loc-{location_segment}__"
f"block-{block_segment}__"
f"chunk-{chunk_size_sqm}__"
f"r{row_index:04d}c{col_index:04d}"
)
def _resolve_boundary(
*,
location: SoilLocation,
boundary: dict | list | None,
block_subdivision: BlockSubdivision | None,
) -> dict | list:
if boundary:
return boundary
if block_subdivision is not None and block_subdivision.source_boundary:
return block_subdivision.source_boundary
if location.farm_boundary:
return location.farm_boundary
raise ValueError("هیچ boundary معتبری برای ساخت analysis grid پیدا نشد.")
def _cell_intersects_polygon(
cell_polygon: list[tuple[float, float]],
polygon: list[tuple[float, float]],
) -> bool:
if any(point_in_polygon(point, polygon) for point in cell_polygon):
return True
for polygon_point in polygon:
if _point_in_rect(polygon_point, cell_polygon):
return True
cell_edges = _polygon_edges(cell_polygon)
polygon_edges = _polygon_edges(polygon)
for edge_a in cell_edges:
for edge_b in polygon_edges:
if _segments_intersect(edge_a[0], edge_a[1], edge_b[0], edge_b[1]):
return True
return False
def _point_in_rect(point: tuple[float, float], rect: list[tuple[float, float]]) -> bool:
xs = [vertex[0] for vertex in rect]
ys = [vertex[1] for vertex in rect]
return min(xs) <= point[0] <= max(xs) and min(ys) <= point[1] <= max(ys)
def _polygon_edges(points: list[tuple[float, float]]) -> list[tuple[tuple[float, float], tuple[float, float]]]:
closed = points + [points[0]]
return [
(closed[index], closed[index + 1])
for index in range(len(points))
]
def _segments_intersect(
p1: tuple[float, float],
p2: tuple[float, float],
q1: tuple[float, float],
q2: tuple[float, float],
) -> bool:
o1 = _orientation(p1, p2, q1)
o2 = _orientation(p1, p2, q2)
o3 = _orientation(q1, q2, p1)
o4 = _orientation(q1, q2, p2)
if o1 != o2 and o3 != o4:
return True
if o1 == 0 and _on_segment(p1, q1, p2):
return True
if o2 == 0 and _on_segment(p1, q2, p2):
return True
if o3 == 0 and _on_segment(q1, p1, q2):
return True
if o4 == 0 and _on_segment(q1, p2, q2):
return True
return False
def _orientation(a: tuple[float, float], b: tuple[float, float], c: tuple[float, float]) -> int:
value = ((b[1] - a[1]) * (c[0] - b[0])) - ((b[0] - a[0]) * (c[1] - b[1]))
if abs(value) < 1e-9:
return 0
return 1 if value > 0 else 2
def _on_segment(a: tuple[float, float], b: tuple[float, float], c: tuple[float, float]) -> bool:
return (
min(a[0], c[0]) <= b[0] <= max(a[0], c[0])
and min(a[1], c[1]) <= b[1] <= max(a[1], c[1])
)
def _update_grid_summary_metadata(
*,
location: SoilLocation,
block_code: str,
chunk_size_sqm: int,
total_count: int,
block_subdivision: BlockSubdivision | None,
) -> None:
if block_subdivision is not None:
metadata = dict(block_subdivision.metadata or {})
metadata["analysis_grid"] = {
"chunk_size_sqm": chunk_size_sqm,
"cell_count": total_count,
}
block_subdivision.metadata = metadata
block_subdivision.save(update_fields=["metadata", "updated_at"])
layout = dict(location.block_layout or {})
blocks = list(layout.get("blocks") or [])
for block in blocks:
if block.get("block_code") == block_code:
block["analysis_grid_summary"] = {
"chunk_size_sqm": chunk_size_sqm,
"cell_count": total_count,
}
break
else:
if not block_code:
layout["analysis_grid_summary"] = {
"chunk_size_sqm": chunk_size_sqm,
"cell_count": total_count,
}
if blocks:
layout["blocks"] = blocks
location.block_layout = layout
location.save(update_fields=["block_layout", "updated_at"])
@@ -1,107 +0,0 @@
"""
Management command to seed a fixed demo farm center location and soil depths.
Run: python manage.py seed_location_data
"""
from django.core.management.base import BaseCommand
from location_data.models import SoilDepthData, SoilLocation
DEMO_LATITUDE = "50.000000"
DEMO_LONGITUDE = "50.000000"
DEMO_BOUNDARY = {
"type": "Polygon",
"coordinates": [
[
[49.995, 49.995],
[50.005, 49.995],
[50.005, 50.005],
[49.995, 50.005],
[49.995, 49.995],
]
],
}
DEMO_SOIL_DEPTHS = {
SoilDepthData.DEPTH_0_5: {
"bdod": 1.22,
"cec": 18.4,
"cfvo": 3.0,
"clay": 24.0,
"nitrogen": 0.21,
"ocd": 26.0,
"ocs": 4.1,
"phh2o": 6.7,
"sand": 38.0,
"silt": 38.0,
"soc": 1.8,
"wv0010": 0.32,
"wv0033": 0.24,
"wv1500": 0.12,
},
SoilDepthData.DEPTH_5_15: {
"bdod": 1.28,
"cec": 17.2,
"cfvo": 4.0,
"clay": 26.0,
"nitrogen": 0.18,
"ocd": 23.0,
"ocs": 3.6,
"phh2o": 6.8,
"sand": 36.0,
"silt": 38.0,
"soc": 1.5,
"wv0010": 0.29,
"wv0033": 0.22,
"wv1500": 0.11,
},
SoilDepthData.DEPTH_15_30: {
"bdod": 1.34,
"cec": 15.9,
"cfvo": 5.0,
"clay": 28.0,
"nitrogen": 0.14,
"ocd": 19.0,
"ocs": 2.9,
"phh2o": 6.9,
"sand": 34.0,
"silt": 38.0,
"soc": 1.2,
"wv0010": 0.26,
"wv0033": 0.19,
"wv1500": 0.09,
},
}
class Command(BaseCommand):
help = "Seed a fixed center location at 50.00, 50.00 plus three soil depth rows."
def handle(self, *args, **options):
location, created = SoilLocation.objects.update_or_create(
latitude=DEMO_LATITUDE,
longitude=DEMO_LONGITUDE,
defaults={
"task_id": "",
"farm_boundary": DEMO_BOUNDARY,
},
)
status_text = "Created" if created else "Updated"
self.stdout.write(
self.style.SUCCESS(
f"{status_text} SoilLocation id={location.id} at ({location.latitude}, {location.longitude})"
)
)
for depth_label, values in DEMO_SOIL_DEPTHS.items():
_, depth_created = SoilDepthData.objects.update_or_create(
soil_location=location,
depth_label=depth_label,
defaults=values,
)
depth_status = "Created" if depth_created else "Updated"
self.stdout.write(
self.style.SUCCESS(f" {depth_status} SoilDepthData {depth_label}")
)
self.stdout.write(self.style.SUCCESS("\nDone seeding location_data demo records."))
@@ -0,0 +1,45 @@
from django.db import migrations, models
def build_default_layout():
return {
"input_block_count": 1,
"default_full_farm": True,
"algorithm_status": "pending",
"blocks": [
{
"block_code": "block-1",
"order": 1,
"source": "default",
"needs_subdivision": None,
"sub_blocks": [],
}
],
}
class Migration(migrations.Migration):
dependencies = [
("location_data", "0007_ndviobservation"),
]
operations = [
migrations.AddField(
model_name="soillocation",
name="block_layout",
field=models.JSONField(
blank=True,
default=build_default_layout,
help_text="ساختار بلوک‌های زمین. به‌صورت پیش‌فرض کل زمین یک بلوک است و بعداً الگوریتم می‌تواند برای هر بلوک زیر‌بلوک تعریف کند.",
),
),
migrations.AddField(
model_name="soillocation",
name="input_block_count",
field=models.PositiveIntegerField(
default=1,
help_text="تعداد بلوک‌های اولیه‌ای که کشاورز برای زمین ثبت می‌کند.",
),
),
]
@@ -0,0 +1,38 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("location_data", "0008_soillocation_block_layout"),
]
operations = [
migrations.CreateModel(
name="BlockSubdivision",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("block_code", models.CharField(help_text="شناسه بلوکی که این خردسازی برای آن انجام شده است.", max_length=64)),
("source_boundary", models.JSONField(blank=True, default=dict, help_text="مرز همان بلوکی که به سرویس subdivision داده شده است.")),
("chunk_size_sqm", models.PositiveIntegerField(default=100, help_text="اندازه هر chunk به متر مربع.")),
("grid_points", models.JSONField(blank=True, default=list, help_text="نقاط اولیه شبکه داخل مرز بلوک.")),
("centroid_points", models.JSONField(blank=True, default=list, help_text="مراکز نهایی بخش‌های خردشده.")),
("grid_point_count", models.PositiveIntegerField(default=0)),
("centroid_count", models.PositiveIntegerField(default=0)),
("status", models.CharField(default="created", help_text="وضعیت تولید subdivision برای این بلوک.", max_length=32)),
("metadata", models.JSONField(blank=True, default=dict)),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("soil_location", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="block_subdivisions", to="location_data.soillocation")),
],
options={
"ordering": ["soil_location", "block_code", "-updated_at"],
"verbose_name": "خردسازی بلوک",
"verbose_name_plural": "خردسازی بلوک‌ها",
},
),
migrations.AddConstraint(
model_name="blocksubdivision",
constraint=models.UniqueConstraint(fields=("soil_location", "block_code"), name="location_block_subdivision_unique_location_block_code"),
),
]
@@ -0,0 +1,21 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("location_data", "0009_blocksubdivision"),
]
operations = [
migrations.AddField(
model_name="blocksubdivision",
name="elbow_plot",
field=models.ImageField(
blank=True,
help_text="تصویر نمودار elbow برای انتخاب تعداد بهینه خوشه‌ها.",
null=True,
upload_to="location_data/elbow_plots/",
),
),
]
@@ -0,0 +1,110 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("location_data", "0010_blocksubdivision_elbow_plot"),
]
operations = [
migrations.CreateModel(
name="AnalysisGridCell",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("block_code", models.CharField(blank=True, db_index=True, default="", help_text="شناسه بلوکی که این سلول به آن تعلق دارد.", max_length=64)),
("cell_code", models.CharField(help_text="شناسه یکتای سلول تحلیل.", max_length=128, unique=True)),
("chunk_size_sqm", models.PositiveIntegerField(db_index=True, default=900, help_text="اندازه سلول تحلیل به متر مربع.")),
("geometry", models.JSONField(blank=True, default=dict, help_text="هندسه سلول به صورت GeoJSON polygon یا ساختار مشابه.")),
("centroid_lat", models.DecimalField(db_index=True, decimal_places=6, help_text="عرض جغرافیایی مرکز سلول.", max_digits=9)),
("centroid_lon", models.DecimalField(db_index=True, decimal_places=6, help_text="طول جغرافیایی مرکز سلول.", max_digits=9)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("block_subdivision", models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, related_name="analysis_grid_cells", to="location_data.blocksubdivision")),
("soil_location", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="analysis_grid_cells", to="location_data.soillocation")),
],
options={
"verbose_name": "analysis grid cell",
"verbose_name_plural": "analysis grid cells",
"ordering": ["soil_location", "block_code", "cell_code"],
},
),
migrations.CreateModel(
name="RemoteSensingRun",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("block_code", models.CharField(blank=True, db_index=True, default="", help_text="شناسه بلوکی که این run برای آن اجرا شده است.", max_length=64)),
("provider", models.CharField(default="openeo", help_text="ارائه‌دهنده داده سنجش‌ازدور.", max_length=64)),
("chunk_size_sqm", models.PositiveIntegerField(default=900, help_text="اندازه هر سلول تحلیل به متر مربع.")),
("temporal_start", models.DateField(blank=True, null=True)),
("temporal_end", models.DateField(blank=True, null=True)),
("status", models.CharField(choices=[("pending", "Pending"), ("running", "Running"), ("success", "Success"), ("failure", "Failure")], db_index=True, default="pending", max_length=16)),
("metadata", models.JSONField(blank=True, default=dict)),
("error_message", models.TextField(blank=True, default="")),
("started_at", models.DateTimeField(blank=True, null=True)),
("finished_at", models.DateTimeField(blank=True, null=True)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("block_subdivision", models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, related_name="remote_sensing_runs", to="location_data.blocksubdivision")),
("soil_location", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="remote_sensing_runs", to="location_data.soillocation")),
],
options={
"verbose_name": "remote sensing run",
"verbose_name_plural": "remote sensing runs",
"ordering": ["-created_at", "-id"],
},
),
migrations.CreateModel(
name="AnalysisGridObservation",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("temporal_start", models.DateField(db_index=True)),
("temporal_end", models.DateField(db_index=True)),
("ndvi", models.FloatField(blank=True, null=True)),
("ndwi", models.FloatField(blank=True, null=True)),
("lst_c", models.FloatField(blank=True, null=True)),
("soil_vv", models.FloatField(blank=True, null=True)),
("soil_vv_db", models.FloatField(blank=True, null=True)),
("dem_m", models.FloatField(blank=True, null=True)),
("slope_deg", models.FloatField(blank=True, null=True)),
("metadata", models.JSONField(blank=True, default=dict)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("cell", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="observations", to="location_data.analysisgridcell")),
("run", models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, related_name="observations", to="location_data.remotesensingrun")),
],
options={
"verbose_name": "analysis grid observation",
"verbose_name_plural": "analysis grid observations",
"ordering": ["-temporal_start", "-temporal_end", "-id"],
},
),
migrations.AddIndex(
model_name="analysisgridcell",
index=models.Index(fields=["soil_location", "block_code"], name="grid_cell_loc_block_idx"),
),
migrations.AddIndex(
model_name="analysisgridcell",
index=models.Index(fields=["soil_location", "chunk_size_sqm"], name="grid_cell_loc_chunk_idx"),
),
migrations.AddIndex(
model_name="remotesensingrun",
index=models.Index(fields=["soil_location", "status", "created_at"], name="rs_run_loc_status_created_idx"),
),
migrations.AddIndex(
model_name="remotesensingrun",
index=models.Index(fields=["block_code", "created_at"], name="rs_run_block_created_idx"),
),
migrations.AddConstraint(
model_name="analysisgridobservation",
constraint=models.UniqueConstraint(fields=("cell", "temporal_start", "temporal_end"), name="grid_obs_unique_cell_temporal_range"),
),
migrations.AddIndex(
model_name="analysisgridobservation",
index=models.Index(fields=["cell", "temporal_start", "temporal_end"], name="grid_obs_cell_temporal_idx"),
),
migrations.AddIndex(
model_name="analysisgridobservation",
index=models.Index(fields=["temporal_start", "temporal_end"], name="grid_obs_temporal_idx"),
),
]
@@ -0,0 +1,65 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("location_data", "0011_remote_sensing_models"),
]
operations = [
migrations.CreateModel(
name="RemoteSensingSubdivisionResult",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("block_code", models.CharField(blank=True, db_index=True, default="", max_length=64)),
("chunk_size_sqm", models.PositiveIntegerField(default=900)),
("temporal_start", models.DateField(db_index=True)),
("temporal_end", models.DateField(db_index=True)),
("cluster_count", models.PositiveIntegerField(default=0)),
("selected_features", models.JSONField(blank=True, default=list)),
("skipped_cell_codes", models.JSONField(blank=True, default=list)),
("metadata", models.JSONField(blank=True, default=dict)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("block_subdivision", models.ForeignKey(blank=True, null=True, on_delete=models.deletion.SET_NULL, related_name="remote_sensing_subdivision_results", to="location_data.blocksubdivision")),
("run", models.OneToOneField(on_delete=models.deletion.CASCADE, related_name="subdivision_result", to="location_data.remotesensingrun")),
("soil_location", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="remote_sensing_subdivision_results", to="location_data.soillocation")),
],
options={
"verbose_name": "remote sensing subdivision result",
"verbose_name_plural": "remote sensing subdivision results",
"ordering": ["-created_at", "-id"],
},
),
migrations.CreateModel(
name="RemoteSensingClusterAssignment",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("cluster_label", models.PositiveIntegerField(db_index=True)),
("raw_feature_values", models.JSONField(blank=True, default=dict)),
("scaled_feature_values", models.JSONField(blank=True, default=dict)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("cell", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="cluster_assignments", to="location_data.analysisgridcell")),
("result", models.ForeignKey(on_delete=models.deletion.CASCADE, related_name="assignments", to="location_data.remotesensingsubdivisionresult")),
],
options={
"verbose_name": "remote sensing cluster assignment",
"verbose_name_plural": "remote sensing cluster assignments",
"ordering": ["cluster_label", "cell__cell_code"],
},
),
migrations.AddIndex(
model_name="remotesensingsubdivisionresult",
index=models.Index(fields=["soil_location", "block_code", "temporal_start", "temporal_end"], name="rs_subdiv_result_lookup_idx"),
),
migrations.AddConstraint(
model_name="remotesensingclusterassignment",
constraint=models.UniqueConstraint(fields=("result", "cell"), name="rs_cluster_assign_unique_result_cell"),
),
migrations.AddIndex(
model_name="remotesensingclusterassignment",
index=models.Index(fields=["result", "cluster_label"], name="rs_cluster_assign_result_label_idx"),
),
]
@@ -0,0 +1,14 @@
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("location_data", "0012_remote_sensing_subdivision_models"),
]
operations = [
migrations.DeleteModel(
name="SoilDepthData",
),
]
@@ -0,0 +1,15 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("location_data", "0013_remove_soildepthdata"),
]
operations = [
migrations.AlterField(
model_name="blocksubdivision",
name="chunk_size_sqm",
field=models.PositiveIntegerField(default=900, help_text="اندازه هر chunk به متر مربع."),
),
]
+414 -41
View File
@@ -1,10 +1,47 @@
from django.db import models
def build_block_layout(block_count: int = 1, blocks: list[dict] | None = None) -> dict:
normalized_blocks = []
if blocks:
for index, block in enumerate(blocks):
normalized_blocks.append(
{
"block_code": str(block.get("block_code") or f"block-{index + 1}").strip(),
"order": int(block.get("order") or index + 1),
"source": "input",
"boundary": block.get("boundary") or {},
"needs_subdivision": None,
"sub_blocks": [],
}
)
else:
normalized_count = max(int(block_count or 1), 1)
for index in range(normalized_count):
normalized_blocks.append(
{
"block_code": f"block-{index + 1}",
"order": index + 1,
"source": "input" if normalized_count > 1 else "default",
"boundary": {},
"needs_subdivision": None,
"sub_blocks": [],
}
)
normalized_count = len(normalized_blocks) if normalized_blocks else max(int(block_count or 1), 1)
return {
"input_block_count": normalized_count,
"default_full_farm": normalized_count == 1,
"algorithm_status": "pending",
"blocks": normalized_blocks,
}
class SoilLocation(models.Model):
"""
مرکز زمین برای داده‌های خاک و مزرعه.
هر مختصات سه سطر در SoilDepthData دارد (۰–۵، ۵–۱۵، ۱۵–۳۰ سانتی‌متر).
مرکز زمین و مرز مزرعه/بلوک‌های تعریف‌شده توسط کشاورز.
"""
latitude = models.DecimalField(
@@ -33,6 +70,18 @@ class SoilLocation(models.Model):
'می‌تواند GeoJSON polygon یا bbox مثل {"type": "Polygon", "coordinates": [...]} باشد.'
),
)
input_block_count = models.PositiveIntegerField(
default=1,
help_text="تعداد بلوک‌های اولیه‌ای که کشاورز برای زمین ثبت می‌کند.",
)
block_layout = models.JSONField(
default=build_block_layout,
blank=True,
help_text=(
"ساختار بلوک‌های زمین. به‌صورت پیش‌فرض کل زمین یک بلوک است و "
"بعداً الگوریتم می‌تواند برای هر بلوک زیر‌بلوک تعریف کند."
),
)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
@@ -60,63 +109,387 @@ class SoilLocation(models.Model):
@property
def is_complete(self):
"""آیا هر سه عمق ذخیره شده‌اند؟"""
return self.depths.count() == 3
"""آیا حداقل یک run کامل remote sensing برای این location وجود دارد؟"""
return self.remote_sensing_runs.filter(status="success").exists()
def set_input_block_count(self, block_count: int = 1, blocks: list[dict] | None = None):
normalized_count = len(blocks) if blocks else max(int(block_count or 1), 1)
self.input_block_count = normalized_count
self.block_layout = build_block_layout(normalized_count, blocks=blocks)
def save(self, *args, **kwargs):
if not self.input_block_count:
self.input_block_count = 1
if not self.block_layout:
self.block_layout = build_block_layout(self.input_block_count)
super().save(*args, **kwargs)
class SoilDepthData(models.Model):
class BlockSubdivision(models.Model):
"""
داده‌های خاک برای یک عمق مشخص، مرتبط با یک SoilLocation.
مقادیر خام از API SoilGrids (قبل از اعمال d_factor).
نتیجه خردسازی یک بلوک برای یک SoilLocation.
grid_points نقاط اولیه شبکه هستند و centroid_points مراکز نهایی بخش‌ها.
"""
DEPTH_0_5 = "0-5cm"
DEPTH_5_15 = "5-15cm"
DEPTH_15_30 = "15-30cm"
DEPTH_CHOICES = [
(DEPTH_0_5, "۰–۵ سانتی‌متر"),
(DEPTH_5_15, "۵–۱۵ سانتی‌متر"),
(DEPTH_15_30, "۱۵–۳۰ سانتی‌متر"),
soil_location = models.ForeignKey(
SoilLocation,
on_delete=models.CASCADE,
related_name="block_subdivisions",
)
block_code = models.CharField(
max_length=64,
help_text="شناسه بلوکی که این خردسازی برای آن انجام شده است.",
)
source_boundary = models.JSONField(
default=dict,
blank=True,
help_text="مرز همان بلوکی که به سرویس subdivision داده شده است.",
)
chunk_size_sqm = models.PositiveIntegerField(
default=900,
help_text="اندازه هر chunk به متر مربع.",
)
grid_points = models.JSONField(
default=list,
blank=True,
help_text="نقاط اولیه شبکه داخل مرز بلوک.",
)
centroid_points = models.JSONField(
default=list,
blank=True,
help_text="مراکز نهایی بخش‌های خردشده.",
)
grid_point_count = models.PositiveIntegerField(default=0)
centroid_count = models.PositiveIntegerField(default=0)
elbow_plot = models.ImageField(
upload_to="location_data/elbow_plots/",
null=True,
blank=True,
help_text="تصویر نمودار elbow برای انتخاب تعداد بهینه خوشه‌ها.",
)
status = models.CharField(
max_length=32,
default="created",
help_text="وضعیت تولید subdivision برای این بلوک.",
)
metadata = models.JSONField(default=dict, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
constraints = [
models.UniqueConstraint(
fields=["soil_location", "block_code"],
name="location_block_subdivision_unique_location_block_code",
)
]
ordering = ["soil_location", "block_code", "-updated_at"]
verbose_name = "خردسازی بلوک"
verbose_name_plural = "خردسازی بلوک‌ها"
def __str__(self):
return f"BlockSubdivision({self.soil_location_id}, {self.block_code})"
class RemoteSensingRun(models.Model):
STATUS_PENDING = "pending"
STATUS_RUNNING = "running"
STATUS_SUCCESS = "success"
STATUS_FAILURE = "failure"
STATUS_CHOICES = [
(STATUS_PENDING, "Pending"),
(STATUS_RUNNING, "Running"),
(STATUS_SUCCESS, "Success"),
(STATUS_FAILURE, "Failure"),
]
soil_location = models.ForeignKey(
SoilLocation,
on_delete=models.CASCADE,
related_name="depths",
related_name="remote_sensing_runs",
)
depth_label = models.CharField(
max_length=10,
choices=DEPTH_CHOICES,
block_subdivision = models.ForeignKey(
BlockSubdivision,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="remote_sensing_runs",
)
block_code = models.CharField(
max_length=64,
blank=True,
default="",
db_index=True,
help_text="شناسه بلوکی که این run برای آن اجرا شده است.",
)
provider = models.CharField(
max_length=64,
default="openeo",
help_text="ارائه‌دهنده داده سنجش‌ازدور.",
)
chunk_size_sqm = models.PositiveIntegerField(
default=900,
help_text="اندازه هر سلول تحلیل به متر مربع.",
)
temporal_start = models.DateField(null=True, blank=True)
temporal_end = models.DateField(null=True, blank=True)
status = models.CharField(
max_length=16,
choices=STATUS_CHOICES,
default=STATUS_PENDING,
db_index=True,
)
# خواص خاک — مقادیر mean از API (raw)
bdod = models.FloatField(null=True, blank=True)
cec = models.FloatField(null=True, blank=True)
cfvo = models.FloatField(null=True, blank=True)
clay = models.FloatField(null=True, blank=True)
nitrogen = models.FloatField(null=True, blank=True)
ocd = models.FloatField(null=True, blank=True)
ocs = models.FloatField(null=True, blank=True)
phh2o = models.FloatField(null=True, blank=True)
sand = models.FloatField(null=True, blank=True)
silt = models.FloatField(null=True, blank=True)
soc = models.FloatField(null=True, blank=True)
wv0010 = models.FloatField(null=True, blank=True)
wv0033 = models.FloatField(null=True, blank=True)
wv1500 = models.FloatField(null=True, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
metadata = models.JSONField(default=dict, blank=True)
error_message = models.TextField(blank=True, default="")
started_at = models.DateTimeField(null=True, blank=True)
finished_at = models.DateTimeField(null=True, blank=True)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
constraints = [
models.UniqueConstraint(
fields=["soil_location", "depth_label"],
name="soil_depth_unique_location_depth",
)
ordering = ["-created_at", "-id"]
indexes = [
models.Index(
fields=["soil_location", "status", "created_at"],
name="rs_run_loc_status_created_idx",
),
models.Index(
fields=["block_code", "created_at"],
name="rs_run_block_created_idx",
),
]
ordering = ["soil_location", "depth_label"]
verbose_name = "remote sensing run"
verbose_name_plural = "remote sensing runs"
def __str__(self):
return f"SoilDepthData({self.soil_location_id}, {self.depth_label})"
block_text = self.block_code or "farm"
return f"RemoteSensingRun({self.soil_location_id}, {block_text}, {self.status})"
@property
def normalized_status(self) -> str:
"""
Return the client-facing lifecycle status while keeping legacy DB values stable.
"""
if self.status == self.STATUS_SUCCESS:
return "completed"
if self.status == self.STATUS_FAILURE:
return "failed"
return self.status
class AnalysisGridCell(models.Model):
soil_location = models.ForeignKey(
SoilLocation,
on_delete=models.CASCADE,
related_name="analysis_grid_cells",
)
block_subdivision = models.ForeignKey(
BlockSubdivision,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="analysis_grid_cells",
)
block_code = models.CharField(
max_length=64,
blank=True,
default="",
db_index=True,
help_text="شناسه بلوکی که این سلول به آن تعلق دارد.",
)
cell_code = models.CharField(
max_length=128,
unique=True,
help_text="شناسه یکتای سلول تحلیل.",
)
chunk_size_sqm = models.PositiveIntegerField(
default=900,
db_index=True,
help_text="اندازه سلول تحلیل به متر مربع.",
)
geometry = models.JSONField(
default=dict,
blank=True,
help_text="هندسه سلول به صورت GeoJSON polygon یا ساختار مشابه.",
)
centroid_lat = models.DecimalField(
max_digits=9,
decimal_places=6,
db_index=True,
help_text="عرض جغرافیایی مرکز سلول.",
)
centroid_lon = models.DecimalField(
max_digits=9,
decimal_places=6,
db_index=True,
help_text="طول جغرافیایی مرکز سلول.",
)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["soil_location", "block_code", "cell_code"]
indexes = [
models.Index(
fields=["soil_location", "block_code"],
name="grid_cell_loc_block_idx",
),
models.Index(
fields=["soil_location", "chunk_size_sqm"],
name="grid_cell_loc_chunk_idx",
),
]
verbose_name = "analysis grid cell"
verbose_name_plural = "analysis grid cells"
def __str__(self):
return f"AnalysisGridCell({self.cell_code})"
class AnalysisGridObservation(models.Model):
cell = models.ForeignKey(
AnalysisGridCell,
on_delete=models.CASCADE,
related_name="observations",
)
run = models.ForeignKey(
RemoteSensingRun,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="observations",
)
temporal_start = models.DateField(db_index=True)
temporal_end = models.DateField(db_index=True)
ndvi = models.FloatField(null=True, blank=True)
ndwi = models.FloatField(null=True, blank=True)
lst_c = models.FloatField(null=True, blank=True)
soil_vv = models.FloatField(null=True, blank=True)
soil_vv_db = models.FloatField(null=True, blank=True)
dem_m = models.FloatField(null=True, blank=True)
slope_deg = models.FloatField(null=True, blank=True)
metadata = models.JSONField(default=dict, blank=True)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["-temporal_start", "-temporal_end", "-id"]
constraints = [
models.UniqueConstraint(
fields=["cell", "temporal_start", "temporal_end"],
name="grid_obs_unique_cell_temporal_range",
)
]
indexes = [
models.Index(
fields=["cell", "temporal_start", "temporal_end"],
name="grid_obs_cell_temporal_idx",
),
models.Index(
fields=["temporal_start", "temporal_end"],
name="grid_obs_temporal_idx",
),
]
verbose_name = "analysis grid observation"
verbose_name_plural = "analysis grid observations"
def __str__(self):
return (
f"AnalysisGridObservation({self.cell_id}, "
f"{self.temporal_start}, {self.temporal_end})"
)
class RemoteSensingSubdivisionResult(models.Model):
soil_location = models.ForeignKey(
SoilLocation,
on_delete=models.CASCADE,
related_name="remote_sensing_subdivision_results",
)
run = models.OneToOneField(
RemoteSensingRun,
on_delete=models.CASCADE,
related_name="subdivision_result",
)
block_subdivision = models.ForeignKey(
BlockSubdivision,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="remote_sensing_subdivision_results",
)
block_code = models.CharField(
max_length=64,
blank=True,
default="",
db_index=True,
)
chunk_size_sqm = models.PositiveIntegerField(default=900)
temporal_start = models.DateField(db_index=True)
temporal_end = models.DateField(db_index=True)
cluster_count = models.PositiveIntegerField(default=0)
selected_features = models.JSONField(default=list, blank=True)
skipped_cell_codes = models.JSONField(default=list, blank=True)
metadata = models.JSONField(default=dict, blank=True)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["-created_at", "-id"]
indexes = [
models.Index(
fields=["soil_location", "block_code", "temporal_start", "temporal_end"],
name="rs_subdiv_result_lookup_idx",
)
]
verbose_name = "remote sensing subdivision result"
verbose_name_plural = "remote sensing subdivision results"
def __str__(self):
return (
f"RemoteSensingSubdivisionResult({self.soil_location_id}, "
f"{self.block_code or 'farm'}, clusters={self.cluster_count})"
)
class RemoteSensingClusterAssignment(models.Model):
result = models.ForeignKey(
RemoteSensingSubdivisionResult,
on_delete=models.CASCADE,
related_name="assignments",
)
cell = models.ForeignKey(
AnalysisGridCell,
on_delete=models.CASCADE,
related_name="cluster_assignments",
)
cluster_label = models.PositiveIntegerField(db_index=True)
raw_feature_values = models.JSONField(default=dict, blank=True)
scaled_feature_values = models.JSONField(default=dict, blank=True)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
ordering = ["cluster_label", "cell__cell_code"]
constraints = [
models.UniqueConstraint(
fields=["result", "cell"],
name="rs_cluster_assign_unique_result_cell",
)
]
indexes = [
models.Index(
fields=["result", "cluster_label"],
name="rs_cluster_assign_result_label_idx",
)
]
verbose_name = "remote sensing cluster assignment"
verbose_name_plural = "remote sensing cluster assignments"
def __str__(self):
return f"RemoteSensingClusterAssignment({self.result_id}, {self.cell_id}, {self.cluster_label})"
class NdviObservation(models.Model):
+476
View File
@@ -0,0 +1,476 @@
from __future__ import annotations
import math
import os
from dataclasses import dataclass
from datetime import date
from decimal import Decimal
from typing import Any
from .models import AnalysisGridCell
DEFAULT_OPENEO_BACKEND_URL = "https://openeofed.dataspace.copernicus.eu"
DEFAULT_OPENEO_PROVIDER = "openeo"
SENTINEL2_COLLECTION = "SENTINEL2_L2A"
SENTINEL3_LST_COLLECTION = "SENTINEL3_SLSTR_L2_LST"
SENTINEL1_COLLECTION = "SENTINEL1_GRD"
COPERNICUS_DEM_COLLECTION = "COPERNICUS_30"
VALID_SCL_CLASSES = (4, 5, 6)
METRIC_NAMES = (
"ndvi",
"ndwi",
"lst_c",
"soil_vv",
"soil_vv_db",
"dem_m",
"slope_deg",
)
class OpenEOServiceError(Exception):
"""Base exception for openEO service failures."""
class OpenEOAuthenticationError(OpenEOServiceError):
"""Raised when authentication with the openEO backend fails."""
class OpenEOExecutionError(OpenEOServiceError):
"""Raised when a metric process graph can not be executed successfully."""
@dataclass(frozen=True)
class OpenEOConnectionSettings:
backend_url: str = DEFAULT_OPENEO_BACKEND_URL
auth_method: str = "client_credentials"
client_id: str = ""
client_secret: str = ""
provider_id: str = ""
username: str = ""
password: str = ""
allow_interactive_oidc: bool = False
@classmethod
def from_env(cls) -> "OpenEOConnectionSettings":
return cls(
backend_url=os.environ.get("OPENEO_BACKEND_URL", DEFAULT_OPENEO_BACKEND_URL).strip(),
auth_method=os.environ.get("OPENEO_AUTH_METHOD", "client_credentials").strip().lower(),
client_id=os.environ.get("OPENEO_AUTH_CLIENT_ID", "").strip(),
client_secret=os.environ.get("OPENEO_AUTH_CLIENT_SECRET", "").strip(),
provider_id=os.environ.get("OPENEO_AUTH_PROVIDER_ID", "").strip(),
username=os.environ.get("OPENEO_USERNAME", "").strip(),
password=os.environ.get("OPENEO_PASSWORD", "").strip(),
allow_interactive_oidc=os.environ.get("OPENEO_ALLOW_INTERACTIVE_OIDC", "0").strip().lower()
in {"1", "true", "yes", "on"},
)
def connect_openeo(settings: OpenEOConnectionSettings | None = None):
"""
Build an authenticated openEO connection using environment-driven configuration.
Preferred authentication mode in production is OIDC client credentials.
"""
settings = settings or OpenEOConnectionSettings.from_env()
try:
import openeo
except ImportError as exc: # pragma: no cover - runtime dependency guard
raise OpenEOServiceError("The `openeo` Python client is required for remote sensing jobs.") from exc
connection = openeo.connect(settings.backend_url)
try:
if settings.auth_method == "client_credentials":
if not settings.client_id or not settings.client_secret:
raise OpenEOAuthenticationError(
"OPENEO_AUTH_CLIENT_ID and OPENEO_AUTH_CLIENT_SECRET must be configured."
)
auth_kwargs = {
"client_id": settings.client_id,
"client_secret": settings.client_secret,
}
if settings.provider_id:
auth_kwargs["provider_id"] = settings.provider_id
return connection.authenticate_oidc_client_credentials(**auth_kwargs)
if settings.auth_method == "password":
if not settings.username or not settings.password:
raise OpenEOAuthenticationError(
"OPENEO_USERNAME and OPENEO_PASSWORD must be configured for password auth."
)
auth_kwargs = {
"username": settings.username,
"password": settings.password,
}
if settings.provider_id:
auth_kwargs["provider_id"] = settings.provider_id
return connection.authenticate_oidc_resource_owner_password_credentials(**auth_kwargs)
if settings.auth_method == "oidc":
if not settings.allow_interactive_oidc:
raise OpenEOAuthenticationError(
"Interactive OIDC auth is disabled. Use client credentials in Celery workers."
)
auth_kwargs = {}
if settings.provider_id:
auth_kwargs["provider_id"] = settings.provider_id
return connection.authenticate_oidc(**auth_kwargs)
raise OpenEOAuthenticationError(f"Unsupported OPENEO_AUTH_METHOD: {settings.auth_method}")
except Exception as exc:
if isinstance(exc, OpenEOServiceError):
raise
raise OpenEOAuthenticationError(f"Failed to authenticate with openEO backend: {exc}") from exc
def build_feature_collection(cells: list[AnalysisGridCell]) -> dict[str, Any]:
features = []
for cell in cells:
features.append(
{
"type": "Feature",
"id": cell.cell_code,
"properties": {
"cell_code": cell.cell_code,
"block_code": cell.block_code,
"soil_location_id": cell.soil_location_id,
},
"geometry": cell.geometry,
}
)
return {"type": "FeatureCollection", "features": features}
def build_spatial_extent(cells: list[AnalysisGridCell]) -> dict[str, float]:
if not cells:
raise ValueError("At least one analysis grid cell is required.")
west = None
east = None
south = None
north = None
for cell in cells:
coordinates = ((cell.geometry or {}).get("coordinates") or [[]])[0]
for lon, lat in coordinates:
west = lon if west is None else min(west, lon)
east = lon if east is None else max(east, lon)
south = lat if south is None else min(south, lat)
north = lat if north is None else max(north, lat)
return {
"west": float(west),
"south": float(south),
"east": float(east),
"north": float(north),
}
def build_empty_metric_payload() -> dict[str, Any]:
return {metric_name: None for metric_name in METRIC_NAMES}
def initialize_metric_result_map(cells: list[AnalysisGridCell]) -> dict[str, dict[str, Any]]:
return {cell.cell_code: build_empty_metric_payload() for cell in cells}
def compute_remote_sensing_metrics(
cells: list[AnalysisGridCell],
*,
temporal_start: date | str,
temporal_end: date | str,
connection=None,
) -> dict[str, Any]:
"""
Compute all requested remote sensing metrics in batch mode per metric.
Returns a normalized structure keyed by `cell_code`, plus execution metadata
that can be stored by Celery tasks and Django models.
"""
if not cells:
return {
"results": {},
"metadata": {
"backend": DEFAULT_OPENEO_PROVIDER,
"collections_used": [],
"slope_supported": False,
"job_refs": {},
"failed_metrics": [],
},
}
connection = connection or connect_openeo()
feature_collection = build_feature_collection(cells)
spatial_extent = build_spatial_extent(cells)
results = initialize_metric_result_map(cells)
metadata = {
"backend": DEFAULT_OPENEO_PROVIDER,
"backend_url": DEFAULT_OPENEO_BACKEND_URL,
"collections_used": [
SENTINEL2_COLLECTION,
SENTINEL3_LST_COLLECTION,
SENTINEL1_COLLECTION,
COPERNICUS_DEM_COLLECTION,
],
"slope_supported": True,
"job_refs": {},
"failed_metrics": [],
}
metric_runners = [
("ndvi", compute_ndvi),
("ndwi", compute_ndwi),
("lst_c", compute_lst_c),
("soil_vv", compute_soil_vv),
("dem_m", compute_dem_m),
("slope_deg", compute_slope_deg),
]
for metric_name, runner in metric_runners:
try:
metric_payload = runner(
connection=connection,
feature_collection=feature_collection,
spatial_extent=spatial_extent,
temporal_start=temporal_start,
temporal_end=temporal_end,
)
merge_metric_results(results, metric_payload["results"])
metadata["job_refs"][metric_name] = metric_payload.get("job_ref")
if metric_name == "slope_deg" and not metric_payload.get("supported", True):
metadata["slope_supported"] = False
except Exception as exc:
if metric_name == "slope_deg":
metadata["slope_supported"] = False
metadata["failed_metrics"].append(
{"metric": metric_name, "error": str(exc), "non_fatal": True}
)
continue
raise OpenEOExecutionError(f"Failed to compute metric `{metric_name}`: {exc}") from exc
for cell_code, payload in results.items():
soil_vv = payload.get("soil_vv")
payload["soil_vv_db"] = linear_to_db(soil_vv)
return {"results": results, "metadata": metadata}
def compute_ndvi(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
SENTINEL2_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
bands=["B03", "B04", "B08", "SCL"],
)
scl = cube.band("SCL")
invalid_mask = (scl != VALID_SCL_CLASSES[0]) & (scl != VALID_SCL_CLASSES[1]) & (scl != VALID_SCL_CLASSES[2])
red = cube.band("B04") * 0.0001
nir = cube.band("B08") * 0.0001
ndvi = ((nir - red) / (nir + red)).mask(invalid_mask.resample_cube_spatial(red))
aggregated = ndvi.mean_time().aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {"results": parse_aggregate_spatial_response(aggregated, "ndvi")}
def compute_ndwi(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
SENTINEL2_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
bands=["B03", "B08", "SCL"],
)
scl = cube.band("SCL")
invalid_mask = (scl != VALID_SCL_CLASSES[0]) & (scl != VALID_SCL_CLASSES[1]) & (scl != VALID_SCL_CLASSES[2])
green = cube.band("B03") * 0.0001
nir = cube.band("B08") * 0.0001
ndwi = ((green - nir) / (green + nir)).mask(invalid_mask.resample_cube_spatial(green))
aggregated = ndwi.mean_time().aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {"results": parse_aggregate_spatial_response(aggregated, "ndwi")}
def compute_lst_c(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
SENTINEL3_LST_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
)
band_name = infer_band_name(cube, preferred=("LST", "LST_in", "LST", "band_0"))
lst_k = cube.band(band_name) if band_name else cube
lst_c = lst_k - 273.15
aggregated = lst_c.mean_time().aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {"results": parse_aggregate_spatial_response(aggregated, "lst_c")}
def compute_soil_vv(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
SENTINEL1_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
bands=["VV"],
)
vv = cube.band("VV")
aggregated = vv.mean_time().aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {"results": parse_aggregate_spatial_response(aggregated, "soil_vv")}
def compute_dem_m(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
COPERNICUS_DEM_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
)
band_name = infer_band_name(cube, preferred=("DEM", "elevation", "band_0"))
dem = cube.band(band_name) if band_name else cube
aggregated = dem.aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {"results": parse_aggregate_spatial_response(aggregated, "dem_m")}
def compute_slope_deg(*, connection, feature_collection, spatial_extent, temporal_start, temporal_end) -> dict[str, Any]:
cube = connection.load_collection(
COPERNICUS_DEM_COLLECTION,
spatial_extent=spatial_extent,
temporal_extent=[_normalize_date(temporal_start), _normalize_date(temporal_end)],
)
band_name = infer_band_name(cube, preferred=("DEM", "elevation", "band_0"))
dem = cube.band(band_name) if band_name else cube
try:
slope_rad = dem.slope()
slope_deg = slope_rad * (180.0 / math.pi)
aggregated = slope_deg.aggregate_spatial(geometries=feature_collection, reducer="mean").execute()
return {
"results": parse_aggregate_spatial_response(aggregated, "slope_deg"),
"supported": True,
}
except Exception:
return {
"results": {feature["id"]: {"slope_deg": None} for feature in feature_collection.get("features", [])},
"supported": False,
}
def parse_aggregate_spatial_response(payload: Any, metric_name: str) -> dict[str, dict[str, Any]]:
"""
Parse different JSON shapes returned by openEO aggregate_spatial executions.
"""
if payload is None:
return {}
if isinstance(payload, dict) and payload.get("type") == "FeatureCollection":
return _parse_feature_collection_results(payload, metric_name)
if isinstance(payload, dict) and "features" in payload:
return _parse_feature_collection_results(payload, metric_name)
if isinstance(payload, dict):
return _parse_mapping_results(payload, metric_name)
if isinstance(payload, list):
return _parse_list_results(payload, metric_name)
raise OpenEOExecutionError(f"Unsupported openEO aggregate_spatial response type: {type(payload)!r}")
def _parse_feature_collection_results(payload: dict[str, Any], metric_name: str) -> dict[str, dict[str, Any]]:
results: dict[str, dict[str, Any]] = {}
for feature in payload.get("features", []):
feature_id = str(
feature.get("id")
or (feature.get("properties") or {}).get("cell_code")
or (feature.get("properties") or {}).get("id")
)
if not feature_id:
continue
properties = feature.get("properties") or {}
value = _extract_aggregate_value(properties)
results[feature_id] = {metric_name: _coerce_float(value)}
return results
def _parse_mapping_results(payload: dict[str, Any], metric_name: str) -> dict[str, dict[str, Any]]:
if "data" in payload and isinstance(payload["data"], (dict, list)):
return parse_aggregate_spatial_response(payload["data"], metric_name)
results: dict[str, dict[str, Any]] = {}
for feature_id, value in payload.items():
if feature_id in {"type", "links", "meta"}:
continue
results[str(feature_id)] = {metric_name: _coerce_float(_extract_aggregate_value(value))}
return results
def _parse_list_results(payload: list[Any], metric_name: str) -> dict[str, dict[str, Any]]:
results: dict[str, dict[str, Any]] = {}
for index, item in enumerate(payload):
if isinstance(item, dict):
feature_id = str(item.get("id") or item.get("cell_code") or item.get("feature_id") or index)
value = _extract_aggregate_value(item)
else:
feature_id = str(index)
value = item
results[feature_id] = {metric_name: _coerce_float(value)}
return results
def _extract_aggregate_value(value: Any) -> Any:
if isinstance(value, dict):
for key in ("mean", "value", "result", "average"):
if key in value:
return _extract_aggregate_value(value[key])
if len(value) == 1:
return _extract_aggregate_value(next(iter(value.values())))
return None
if isinstance(value, list):
if not value:
return None
return _extract_aggregate_value(value[0])
return value
def merge_metric_results(target: dict[str, dict[str, Any]], updates: dict[str, dict[str, Any]]) -> None:
for cell_code, values in updates.items():
target.setdefault(cell_code, build_empty_metric_payload())
target[cell_code].update(values)
def linear_to_db(value: Any) -> float | None:
numeric = _coerce_float(value)
if numeric is None or numeric <= 0:
return None
return round(10.0 * math.log10(numeric), 6)
def infer_band_name(cube, preferred: tuple[str, ...]) -> str | None:
"""
Best-effort band name selection for collections with backend-specific naming.
"""
metadata = getattr(cube, "metadata", None)
if metadata is None:
return None
band_dimension = getattr(metadata, "band_dimension", None)
bands = getattr(band_dimension, "bands", None)
if not bands:
return None
available = []
for band in bands:
name = getattr(band, "name", None) or str(band)
available.append(name)
for candidate in preferred:
if candidate in available:
return candidate
return available[0] if available else None
def _coerce_float(value: Any) -> float | None:
if value is None:
return None
if isinstance(value, Decimal):
return float(value)
try:
return float(value)
except (TypeError, ValueError):
return None
def _normalize_date(value: date | str) -> str:
if isinstance(value, date):
return value.isoformat()
return str(value)
+116
View File
@@ -0,0 +1,116 @@
from __future__ import annotations
from typing import Any
from django.db.models import Avg, QuerySet
from .models import AnalysisGridObservation, RemoteSensingRun, SoilLocation
SATELLITE_METRIC_FIELDS = (
"ndvi",
"ndwi",
"lst_c",
"soil_vv_db",
"dem_m",
"slope_deg",
)
def build_location_satellite_snapshot(
location: SoilLocation,
*,
block_code: str = "",
) -> dict[str, Any]:
run = get_latest_completed_remote_sensing_run(location, block_code=block_code)
if run is None:
return {
"status": "missing",
"block_code": block_code,
"run_id": None,
"temporal_extent": None,
"cell_count": 0,
"resolved_metrics": {},
"metric_sources": {},
}
observations = get_run_observations(run)
summary = summarize_observations(observations)
return {
"status": "completed",
"block_code": run.block_code,
"run_id": run.id,
"temporal_extent": {
"start_date": run.temporal_start.isoformat() if run.temporal_start else None,
"end_date": run.temporal_end.isoformat() if run.temporal_end else None,
},
"cell_count": observations.count(),
"resolved_metrics": summary,
"metric_sources": {
metric_name: "remote_sensing"
for metric_name in summary
},
}
def build_location_block_satellite_snapshots(location: SoilLocation) -> list[dict[str, Any]]:
block_layout = location.block_layout or {}
blocks = block_layout.get("blocks") or []
if not blocks:
return [build_location_satellite_snapshot(location)]
snapshots = []
for block in blocks:
snapshots.append(
build_location_satellite_snapshot(
location,
block_code=str(block.get("block_code") or "").strip(),
)
)
return snapshots
def get_latest_completed_remote_sensing_run(
location: SoilLocation,
*,
block_code: str = "",
) -> RemoteSensingRun | None:
return (
RemoteSensingRun.objects.filter(
soil_location=location,
block_code=block_code or "",
status=RemoteSensingRun.STATUS_SUCCESS,
)
.order_by("-temporal_end", "-created_at", "-id")
.first()
)
def get_run_observations(run: RemoteSensingRun) -> QuerySet[AnalysisGridObservation]:
return (
AnalysisGridObservation.objects.select_related("cell", "run")
.filter(
cell__soil_location=run.soil_location,
cell__block_code=run.block_code or "",
temporal_start=run.temporal_start,
temporal_end=run.temporal_end,
)
.order_by("cell__cell_code")
)
def summarize_observations(
observations: QuerySet[AnalysisGridObservation],
) -> dict[str, float]:
aggregates = observations.aggregate(
**{
f"{metric_name}_mean": Avg(metric_name)
for metric_name in SATELLITE_METRIC_FIELDS
}
)
summary: dict[str, float] = {}
for metric_name in SATELLITE_METRIC_FIELDS:
value = aggregates.get(f"{metric_name}_mean")
if value is None:
continue
summary[metric_name] = round(float(value), 6)
return summary
+280 -36
View File
@@ -1,42 +1,49 @@
from rest_framework import serializers
from .models import SoilDepthData, SoilLocation
from .soil_adapters import DEPTHS
from .data_driven_subdivision import SUPPORTED_CLUSTER_FEATURES
from .models import (
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingRun,
RemoteSensingClusterAssignment,
RemoteSensingSubdivisionResult,
SoilLocation,
)
from .satellite_snapshot import build_location_block_satellite_snapshots
class SoilDataRequestSerializer(serializers.Serializer):
"""سریالایزر ورودی: lon و lat برای درخواست داده خاک."""
"""ورودی ثبت مزرعه و بلوک‌های تعریف‌شده توسط کشاورز."""
class BlockInputSerializer(serializers.Serializer):
block_code = serializers.CharField(max_length=64)
boundary = serializers.JSONField()
order = serializers.IntegerField(required=False, min_value=1)
lon = serializers.DecimalField(max_digits=9, decimal_places=6, required=True)
lat = serializers.DecimalField(max_digits=9, decimal_places=6, required=True)
block_count = serializers.IntegerField(required=False, min_value=1, default=1)
block_code = serializers.CharField(required=False, default="block-1", max_length=64)
farm_boundary = serializers.JSONField(required=False)
blocks = BlockInputSerializer(many=True, required=False)
class SoilDepthDataSerializer(serializers.ModelSerializer):
"""سریالایزر خروجی برای هر عمق خاک."""
class Meta:
model = SoilDepthData
fields = [
"depth_label",
"bdod",
"cec",
"cfvo",
"clay",
"nitrogen",
"ocd",
"ocs",
"phh2o",
"sand",
"silt",
"soc",
"wv0010",
"wv0033",
"wv1500",
]
def validate(self, attrs):
blocks = attrs.get("blocks") or []
if self.context.get("require_farm_boundary") and not attrs.get("farm_boundary"):
raise serializers.ValidationError(
{"farm_boundary": ["مختصات گوشه‌های کل زمین باید ارسال شود."]}
)
if self.context.get("require_farm_boundary") and not blocks:
raise serializers.ValidationError(
{"blocks": ["مختصات بلوک‌های تعریف‌شده توسط کشاورز باید ارسال شود."]}
)
if blocks:
attrs["block_count"] = len(blocks)
return attrs
class SoilLocationResponseSerializer(serializers.ModelSerializer):
"""سریالایزر خروجی برای SoilLocation همراه با depths."""
"""سریالایزر خروجی برای SoilLocation همراه با خلاصه سنجش‌ازدور."""
lon = serializers.DecimalField(
source="longitude",
@@ -50,19 +57,51 @@ class SoilLocationResponseSerializer(serializers.ModelSerializer):
decimal_places=6,
read_only=True,
)
depths = serializers.SerializerMethodField()
input_block_count = serializers.IntegerField(read_only=True)
farm_boundary = serializers.JSONField(read_only=True)
block_layout = serializers.JSONField(read_only=True)
block_subdivisions = serializers.SerializerMethodField()
satellite_snapshots = serializers.SerializerMethodField()
class Meta:
model = SoilLocation
fields = ["id", "lon", "lat", "depths"]
fields = [
"id",
"lon",
"lat",
"input_block_count",
"farm_boundary",
"block_layout",
"block_subdivisions",
"satellite_snapshots",
]
def get_depths(self, obj):
depth_qs = obj.depths.all()
order = {d: i for i, d in enumerate(DEPTHS)}
sorted_depths = sorted(
depth_qs, key=lambda d: order.get(d.depth_label, 99)
)
return SoilDepthDataSerializer(sorted_depths, many=True).data
def get_block_subdivisions(self, obj):
subdivisions = obj.block_subdivisions.all().order_by("block_code", "id")
return BlockSubdivisionSerializer(subdivisions, many=True).data
def get_satellite_snapshots(self, obj):
return build_location_block_satellite_snapshots(obj)
class BlockSubdivisionSerializer(serializers.ModelSerializer):
elbow_plot = serializers.ImageField(read_only=True)
class Meta:
model = BlockSubdivision
fields = [
"block_code",
"chunk_size_sqm",
"grid_points",
"centroid_points",
"grid_point_count",
"centroid_count",
"elbow_plot",
"status",
"metadata",
"created_at",
"updated_at",
]
class SoilDataTaskResponseSerializer(serializers.Serializer):
@@ -94,3 +133,208 @@ class NdviHealthResponseSerializer(serializers.Serializer):
observation_date = serializers.CharField(allow_null=True)
satellite_source = serializers.CharField(allow_null=True)
healthData = NdviHealthDataItemSerializer(many=True)
class RemoteSensingTriggerSerializer(serializers.Serializer):
lon = serializers.DecimalField(max_digits=9, decimal_places=6, required=True)
lat = serializers.DecimalField(max_digits=9, decimal_places=6, required=True)
block_code = serializers.CharField(required=False, allow_blank=True, default="", max_length=64)
start_date = serializers.DateField(required=True)
end_date = serializers.DateField(required=True)
force_refresh = serializers.BooleanField(required=False, default=False)
cluster_count = serializers.IntegerField(required=False, min_value=1, allow_null=True, default=None)
selected_features = serializers.ListField(
child=serializers.CharField(max_length=64),
required=False,
allow_empty=False,
)
def validate(self, attrs):
if attrs["start_date"] > attrs["end_date"]:
raise serializers.ValidationError("start_date نمی‌تواند بعد از end_date باشد.")
selected_features = attrs.get("selected_features") or []
invalid_features = sorted(
feature_name
for feature_name in selected_features
if feature_name not in SUPPORTED_CLUSTER_FEATURES
)
if invalid_features:
raise serializers.ValidationError(
{
"selected_features": [
"ویژگی‌های نامعتبر برای خوشه‌بندی: "
+ ", ".join(invalid_features)
]
}
)
return attrs
class RemoteSensingResultQuerySerializer(RemoteSensingTriggerSerializer):
page = serializers.IntegerField(required=False, min_value=1, default=1)
page_size = serializers.IntegerField(required=False, min_value=1, max_value=200, default=100)
class RemoteSensingCellObservationSerializer(serializers.ModelSerializer):
cell_code = serializers.CharField(source="cell.cell_code", read_only=True)
block_code = serializers.CharField(source="cell.block_code", read_only=True)
chunk_size_sqm = serializers.IntegerField(source="cell.chunk_size_sqm", read_only=True)
centroid_lat = serializers.DecimalField(source="cell.centroid_lat", max_digits=9, decimal_places=6, read_only=True)
centroid_lon = serializers.DecimalField(source="cell.centroid_lon", max_digits=9, decimal_places=6, read_only=True)
geometry = serializers.JSONField(source="cell.geometry", read_only=True)
class Meta:
model = AnalysisGridObservation
fields = [
"cell_code",
"block_code",
"chunk_size_sqm",
"centroid_lat",
"centroid_lon",
"geometry",
"temporal_start",
"temporal_end",
"ndvi",
"ndwi",
"lst_c",
"soil_vv",
"soil_vv_db",
"dem_m",
"slope_deg",
"metadata",
]
class RemoteSensingSummarySerializer(serializers.Serializer):
cell_count = serializers.IntegerField()
ndvi_mean = serializers.FloatField(allow_null=True)
ndwi_mean = serializers.FloatField(allow_null=True)
lst_c_mean = serializers.FloatField(allow_null=True)
soil_vv_db_mean = serializers.FloatField(allow_null=True)
dem_m_mean = serializers.FloatField(allow_null=True)
slope_deg_mean = serializers.FloatField(allow_null=True)
class RemoteSensingRunSerializer(serializers.ModelSerializer):
status_label = serializers.SerializerMethodField()
pipeline_status = serializers.SerializerMethodField()
stage = serializers.SerializerMethodField()
selected_features = serializers.SerializerMethodField()
requested_cluster_count = serializers.SerializerMethodField()
def get_status_label(self, obj):
return obj.normalized_status
def get_pipeline_status(self, obj):
return obj.normalized_status
def get_stage(self, obj):
return (obj.metadata or {}).get("stage")
def get_selected_features(self, obj):
return (obj.metadata or {}).get("selected_features", [])
def get_requested_cluster_count(self, obj):
return (obj.metadata or {}).get("requested_cluster_count")
class Meta:
model = RemoteSensingRun
fields = [
"id",
"block_code",
"chunk_size_sqm",
"temporal_start",
"temporal_end",
"status",
"status_label",
"pipeline_status",
"stage",
"selected_features",
"requested_cluster_count",
"metadata",
"error_message",
"started_at",
"finished_at",
"created_at",
"updated_at",
]
class RemoteSensingClusterAssignmentSerializer(serializers.ModelSerializer):
cell_code = serializers.CharField(source="cell.cell_code", read_only=True)
centroid_lat = serializers.DecimalField(source="cell.centroid_lat", max_digits=9, decimal_places=6, read_only=True)
centroid_lon = serializers.DecimalField(source="cell.centroid_lon", max_digits=9, decimal_places=6, read_only=True)
class Meta:
model = RemoteSensingClusterAssignment
fields = [
"cell_code",
"cluster_label",
"centroid_lat",
"centroid_lon",
"raw_feature_values",
"scaled_feature_values",
]
class RemoteSensingSubdivisionResultSerializer(serializers.ModelSerializer):
assignments = serializers.SerializerMethodField()
def get_assignments(self, obj):
assignments = self.context.get("paginated_assignments")
if assignments is None:
assignments = obj.assignments.all().order_by("cluster_label", "cell__cell_code")
return RemoteSensingClusterAssignmentSerializer(assignments, many=True).data
class Meta:
model = RemoteSensingSubdivisionResult
fields = [
"id",
"block_code",
"chunk_size_sqm",
"temporal_start",
"temporal_end",
"cluster_count",
"selected_features",
"skipped_cell_codes",
"metadata",
"assignments",
"created_at",
"updated_at",
]
class RemoteSensingResponseSerializer(serializers.Serializer):
status = serializers.CharField()
source = serializers.CharField()
location = SoilLocationResponseSerializer()
block_code = serializers.CharField(allow_blank=True)
chunk_size_sqm = serializers.IntegerField(allow_null=True)
temporal_extent = serializers.JSONField()
summary = RemoteSensingSummarySerializer()
cells = RemoteSensingCellObservationSerializer(many=True)
run = RemoteSensingRunSerializer(allow_null=True)
subdivision_result = RemoteSensingSubdivisionResultSerializer(allow_null=True)
pagination = serializers.JSONField(required=False)
class RemoteSensingRunStatusResponseSerializer(serializers.Serializer):
status = serializers.CharField()
source = serializers.CharField()
run = RemoteSensingRunSerializer()
task_id = serializers.CharField(allow_blank=True, allow_null=True, required=False)
class RemoteSensingRunResultResponseSerializer(serializers.Serializer):
status = serializers.CharField()
source = serializers.CharField()
location = SoilLocationResponseSerializer()
block_code = serializers.CharField(allow_blank=True)
chunk_size_sqm = serializers.IntegerField(allow_null=True)
temporal_extent = serializers.JSONField()
summary = RemoteSensingSummarySerializer()
cells = RemoteSensingCellObservationSerializer(many=True)
run = RemoteSensingRunSerializer()
subdivision_result = RemoteSensingSubdivisionResultSerializer(allow_null=True)
pagination = serializers.JSONField(required=False)
-286
View File
@@ -1,286 +0,0 @@
from __future__ import annotations
import hashlib
import math
import random
import time
from abc import ABC, abstractmethod
try:
import requests
except ImportError: # pragma: no cover - handled when live adapter is used
requests = None
SOILGRIDS_BASE = "https://rest.isric.org/soilgrids/v2.0/properties/query"
PROPERTIES = [
"bdod",
"cec",
"cfvo",
"clay",
"nitrogen",
"ocd",
"ocs",
"phh2o",
"sand",
"silt",
"soc",
"wv0010",
"wv0033",
"wv1500",
]
VALUES = ["Q0.5", "Q0.05", "Q0.95", "mean", "uncertainty"]
DEPTHS = ["0-5cm", "5-15cm", "15-30cm"]
DEPTH_INDEX = {depth: index for index, depth in enumerate(DEPTHS)}
def _clamp(value: float, lower: float, upper: float) -> float:
return max(lower, min(upper, value))
def _round_field(name: str, value: float) -> float:
if name in {"nitrogen", "soc", "ocs", "wv0010", "wv0033", "wv1500"}:
return round(value, 3)
return round(value, 2)
class BaseSoilDataAdapter(ABC):
source_name = "base"
@abstractmethod
def fetch_depth_fields(self, lon: float, lat: float, depth: str) -> dict:
"""Return normalized field values for a single soil depth."""
class SoilGridsAdapter(BaseSoilDataAdapter):
source_name = "soilgrids"
def __init__(self, base_url: str = SOILGRIDS_BASE, timeout: float = 60):
self.base_url = base_url
self.timeout = timeout
def fetch_depth_fields(self, lon: float, lat: float, depth: str) -> dict:
if requests is None:
raise RuntimeError("requests package is required for SoilGridsAdapter")
params = {
"lon": lon,
"lat": lat,
"depth": depth,
}
for prop in PROPERTIES:
params.setdefault("property", []).append(prop)
for value in VALUES:
params.setdefault("value", []).append(value)
response = requests.get(
self.base_url,
params=params,
headers={"accept": "application/json"},
timeout=self.timeout,
)
response.raise_for_status()
return self._parse_response_to_fields(response.json())
def _parse_response_to_fields(self, data: dict) -> dict:
fields = {prop: None for prop in PROPERTIES}
layers = data.get("properties", {}).get("layers", [])
for layer in layers:
name = layer.get("name")
if name not in fields:
continue
depths_list = layer.get("depths", [])
if not depths_list:
continue
values = depths_list[0].get("values", {})
mean_value = values.get("mean")
if mean_value is not None:
fields[name] = float(mean_value)
return fields
class MockSoilDataAdapter(BaseSoilDataAdapter):
source_name = "mock"
def __init__(
self,
delay_seconds: float = 0.8,
seed_namespace: str = "croplogic-soil",
):
self.delay_seconds = max(0.0, delay_seconds)
self.seed_namespace = seed_namespace
def fetch_depth_fields(self, lon: float, lat: float, depth: str) -> dict:
if depth not in DEPTH_INDEX:
raise ValueError(f"Unsupported soil depth: {depth}")
if self.delay_seconds:
time.sleep(self.delay_seconds)
depth_index = DEPTH_INDEX[depth]
texture_score = self._layered_noise(lon, lat, "texture")
organic_score = self._layered_noise(lon, lat, "organic")
moisture_score = self._layered_noise(lon, lat, "moisture")
mineral_score = self._layered_noise(lon, lat, "mineral")
stone_score = self._layered_noise(lon, lat, "stone")
ph_score = self._layered_noise(lon, lat, "ph")
sand, clay, silt = self._build_texture(
texture_score=texture_score,
organic_score=organic_score,
depth_index=depth_index,
)
soc = _clamp(
0.7
+ (organic_score * 1.9)
+ (clay * 0.012)
- (depth_index * 0.28)
+ ((1 - moisture_score) * 0.08),
0.45,
4.2,
)
nitrogen = _clamp(
0.04
+ (soc * 0.085)
+ ((1 - (sand / 100.0)) * 0.025)
+ ((2 - depth_index) * 0.008),
0.03,
0.42,
)
ocd = _clamp(
10.0 + (soc * 8.5) + (organic_score * 4.0) - (depth_index * 2.6),
7.0,
46.0,
)
ocs = _clamp(
1.0 + (soc * 1.55) - (depth_index * 0.28) + (organic_score * 0.12),
0.5,
8.5,
)
cec = _clamp(
7.0
+ (clay * 0.33)
+ (soc * 1.7)
+ ((1 - (sand / 100.0)) * 2.6)
+ (mineral_score * 1.4),
5.0,
38.0,
)
cfvo = _clamp(1.0 + (stone_score * 12.0) + (depth_index * 2.4), 0.0, 35.0)
bdod = _clamp(
1.06
+ (sand * 0.0038)
+ (depth_index * 0.06)
- (soc * 0.035)
+ (stone_score * 0.03),
0.95,
1.62,
)
phh2o = _clamp(
6.2
+ ((ph_score - 0.5) * 1.1)
+ (depth_index * 0.08)
- (organic_score * 0.12),
5.6,
8.1,
)
wv1500 = _clamp(
0.05
+ (clay * 0.0016)
+ (soc * 0.012)
- (sand * 0.0003)
+ (depth_index * 0.004),
0.05,
0.22,
)
wv0033 = _clamp(
wv1500 + 0.07 + (clay * 0.0015) + (soc * 0.01) - (sand * 0.0002),
wv1500 + 0.04,
0.38,
)
wv0010 = _clamp(
wv0033 + 0.03 + (soc * 0.006) + (moisture_score * 0.01),
wv0033 + 0.015,
0.48,
)
fields = {
"bdod": bdod,
"cec": cec,
"cfvo": cfvo,
"clay": clay,
"nitrogen": nitrogen,
"ocd": ocd,
"ocs": ocs,
"phh2o": phh2o,
"sand": sand,
"silt": silt,
"soc": soc,
"wv0010": wv0010,
"wv0033": wv0033,
"wv1500": wv1500,
}
return {name: _round_field(name, value) for name, value in fields.items()}
def _build_texture(
self,
texture_score: float,
organic_score: float,
depth_index: int,
) -> tuple[float, float, float]:
sand = _clamp(
30.0
+ (texture_score * 28.0)
+ ((organic_score - 0.5) * 3.5)
- (depth_index * 2.5),
18.0,
72.0,
)
clay = _clamp(
13.0
+ ((1 - texture_score) * 18.0)
+ (depth_index * 5.5)
+ ((organic_score - 0.5) * 2.0),
8.0,
42.0,
)
minimum_silt = 12.0
total = sand + clay
if total > 100.0 - minimum_silt:
excess = total - (100.0 - minimum_silt)
sand -= excess * 0.65
clay -= excess * 0.35
silt = 100.0 - sand - clay
return sand, clay, silt
def _layered_noise(self, lon: float, lat: float, key: str) -> float:
regional = self._smooth_noise(lon, lat, f"{key}:regional", scale=1.7)
local = self._smooth_noise(lon, lat, f"{key}:local", scale=0.32)
micro = self._smooth_noise(lon, lat, f"{key}:micro", scale=0.08)
return _clamp((regional * 0.55) + (local * 0.3) + (micro * 0.15), 0.0, 1.0)
def _smooth_noise(self, lon: float, lat: float, key: str, scale: float) -> float:
grid_x = lon / scale
grid_y = lat / scale
x0 = math.floor(grid_x)
y0 = math.floor(grid_y)
tx = grid_x - x0
ty = grid_y - y0
v00 = self._cell_noise(key, x0, y0)
v10 = self._cell_noise(key, x0 + 1, y0)
v01 = self._cell_noise(key, x0, y0 + 1)
v11 = self._cell_noise(key, x0 + 1, y0 + 1)
tx = tx * tx * (3.0 - (2.0 * tx))
ty = ty * ty * (3.0 - (2.0 * ty))
top = (v00 * (1 - tx)) + (v10 * tx)
bottom = (v01 * (1 - tx)) + (v11 * tx)
return (top * (1 - ty)) + (bottom * ty)
def _cell_noise(self, key: str, grid_x: int, grid_y: int) -> float:
seed_input = f"{self.seed_namespace}:{key}:{grid_x}:{grid_y}"
digest = hashlib.sha256(seed_input.encode("ascii")).digest()
seed = int.from_bytes(digest[:8], "big", signed=False)
return random.Random(seed).random()
+592 -74
View File
@@ -1,15 +1,36 @@
"""
تسک‌های Celery برای واکشی داده‌های خاک.
تسک‌های Celery برای pipeline سنجش‌ازدور و subdivision داده‌محور.
"""
from decimal import Decimal
import logging
from typing import Any
from config.celery import app
from django.apps import apps
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from django.utils.dateparse import parse_date
from .models import SoilDepthData, SoilLocation
from .soil_adapters import DEPTHS
from .data_driven_subdivision import (
DEFAULT_CLUSTER_FEATURES,
DataDrivenSubdivisionError,
create_remote_sensing_subdivision_result,
)
from .grid_analysis import create_or_get_analysis_grid_cells
from .models import (
AnalysisGridCell,
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingRun,
RemoteSensingSubdivisionResult,
SoilLocation,
)
from .openeo_service import (
OpenEOAuthenticationError,
OpenEOExecutionError,
OpenEOServiceError,
compute_remote_sensing_metrics,
)
try:
import requests
@@ -19,79 +40,576 @@ else:
RequestException = requests.RequestException
def fetch_soil_data_for_coordinates(
latitude: float,
longitude: float,
logger = logging.getLogger(__name__)
def run_remote_sensing_analysis(
*,
soil_location_id: int,
block_code: str = "",
temporal_start: Any,
temporal_end: Any,
force_refresh: bool = False,
task_id: str = "",
progress_callback=None,
run_id: int | None = None,
cluster_count: int | None = None,
selected_features: list[str] | None = None,
) -> dict[str, Any]:
"""
اجرای سنکرون تحلیل سنجش‌ازدور برای یک location/block.
این helper برای Celery task و هر orchestration داخلی دیگر قابل استفاده است.
"""
start_date = _normalize_temporal_date(temporal_start, "temporal_start")
end_date = _normalize_temporal_date(temporal_end, "temporal_end")
if start_date > end_date:
raise ValueError("temporal_start نمی‌تواند بعد از temporal_end باشد.")
location = SoilLocation.objects.filter(pk=soil_location_id).first()
if location is None:
raise ValueError(f"SoilLocation با id={soil_location_id} پیدا نشد.")
resolved_block_code = str(block_code or "").strip()
subdivision = _resolve_block_subdivision(location, resolved_block_code)
run = _get_or_create_remote_sensing_run(
run_id=run_id,
location=location,
subdivision=subdivision,
block_code=resolved_block_code,
temporal_start=start_date,
temporal_end=end_date,
task_id=task_id,
cluster_count=cluster_count,
selected_features=selected_features or list(DEFAULT_CLUSTER_FEATURES),
)
_mark_run_running(run)
try:
_record_run_stage(
run,
"preparing_analysis_grid",
{
"block_code": resolved_block_code,
"temporal_extent": {
"start_date": start_date.isoformat(),
"end_date": end_date.isoformat(),
},
},
)
grid_summary = create_or_get_analysis_grid_cells(
location,
block_code=resolved_block_code,
block_subdivision=subdivision,
)
_record_run_stage(run, "analysis_grid_ready", {"grid_summary": grid_summary})
all_cells = _load_grid_cells(location, resolved_block_code)
cells_to_process = _select_cells_for_processing(
all_cells=all_cells,
temporal_start=start_date,
temporal_end=end_date,
force_refresh=force_refresh,
)
_record_run_stage(
run,
"analysis_cells_selected",
{
"cell_selection": {
"total_cell_count": len(all_cells),
"cell_count_to_process": len(cells_to_process),
"existing_cell_count": len(all_cells) - len(cells_to_process),
"force_refresh": force_refresh,
}
},
)
if not cells_to_process:
_record_run_stage(
run,
"using_cached_observations",
{"source": "database"},
)
observations = _load_observations(
location=location,
block_code=resolved_block_code,
temporal_start=start_date,
temporal_end=end_date,
)
subdivision_result = _ensure_subdivision_result(
location=location,
run=run,
subdivision=subdivision,
block_code=resolved_block_code,
observations=observations,
cluster_count=cluster_count,
selected_features=selected_features,
)
_record_run_stage(
run,
"clustering_completed",
_build_clustering_stage_metadata(subdivision_result),
)
summary = {
"status": "completed",
"source": "database",
"run_id": run.id,
"processed_cell_count": 0,
"created_observation_count": 0,
"updated_observation_count": 0,
"existing_observation_count": len(all_cells),
"failed_metric_count": 0,
"chunk_size_sqm": grid_summary["chunk_size_sqm"],
"block_code": resolved_block_code,
"cell_count": len(all_cells),
"subdivision_result_id": getattr(subdivision_result, "id", None),
"cluster_count": getattr(subdivision_result, "cluster_count", 0),
}
_mark_run_success(run, summary)
return summary
_record_run_stage(
run,
"fetching_remote_metrics",
{"requested_cell_count": len(cells_to_process)},
)
remote_payload = compute_remote_sensing_metrics(
cells_to_process,
temporal_start=start_date,
temporal_end=end_date,
)
_record_run_stage(
run,
"remote_metrics_fetched",
{
"failed_metric_count": len(remote_payload["metadata"].get("failed_metrics", [])),
"service_metadata": remote_payload["metadata"],
},
)
upsert_summary = _upsert_grid_observations(
cells=cells_to_process,
run=run,
temporal_start=start_date,
temporal_end=end_date,
metric_payload=remote_payload,
)
_record_run_stage(run, "observations_persisted", upsert_summary)
observations = _load_observations(
location=location,
block_code=resolved_block_code,
temporal_start=start_date,
temporal_end=end_date,
)
subdivision_result = _ensure_subdivision_result(
location=location,
run=run,
subdivision=subdivision,
block_code=resolved_block_code,
observations=observations,
cluster_count=cluster_count,
selected_features=selected_features,
)
_record_run_stage(
run,
"clustering_completed",
_build_clustering_stage_metadata(subdivision_result),
)
summary = {
"status": "completed",
"source": "openeo",
"run_id": run.id,
"processed_cell_count": len(cells_to_process),
"created_observation_count": upsert_summary["created_count"],
"updated_observation_count": upsert_summary["updated_count"],
"existing_observation_count": len(all_cells) - len(cells_to_process),
"failed_metric_count": len(remote_payload["metadata"].get("failed_metrics", [])),
"chunk_size_sqm": grid_summary["chunk_size_sqm"],
"block_code": resolved_block_code,
"cell_count": len(all_cells),
"subdivision_result_id": subdivision_result.id,
"cluster_count": subdivision_result.cluster_count,
}
_mark_run_success(run, summary, remote_payload["metadata"])
logger.info(
"Remote sensing analysis completed",
extra={
"run_id": run.id,
"soil_location_id": location.id,
"block_code": resolved_block_code,
"processed_cell_count": summary["processed_cell_count"],
},
)
return summary
except Exception as exc:
_mark_run_failure(run, str(exc))
raise
@app.task(bind=True, max_retries=3, default_retry_delay=60)
def run_remote_sensing_analysis_task(
self,
soil_location_id: int,
block_code: str = "",
temporal_start: Any = "",
temporal_end: Any = "",
force_refresh: bool = False,
run_id: int | None = None,
cluster_count: int | None = None,
selected_features: list[str] | None = None,
):
"""
واکشی سنکرون داده خاک برای مختصات داده‌شده و ذخیره در DB.
این helper هم توسط Celery task و هم توسط endpointهای sync استفاده می‌شود.
"""
lat = Decimal(str(round(float(latitude), 6)))
lon = Decimal(str(round(float(longitude), 6)))
adapter = apps.get_app_config("location_data").get_soil_data_adapter()
with transaction.atomic():
location, created = SoilLocation.objects.select_for_update().get_or_create(
latitude=lat,
longitude=lon,
defaults={"task_id": task_id},
)
if not created and task_id:
location.task_id = task_id
location.save(update_fields=["task_id"])
for index, depth in enumerate(DEPTHS):
if progress_callback is not None:
progress_callback(
state="PROGRESS",
meta={
"current": index + 1,
"total": len(DEPTHS),
"message": f"در حال واکشی عمق {depth}...",
},
)
fields = adapter.fetch_depth_fields(float(lon), float(lat), depth)
with transaction.atomic():
SoilDepthData.objects.update_or_create(
soil_location=location,
depth_label=depth,
defaults=fields,
)
if task_id:
with transaction.atomic():
location.task_id = ""
location.save(update_fields=["task_id"])
return {
"status": "completed",
"location_id": location.id,
"depths": DEPTHS,
}
@app.task(bind=True)
def fetch_soil_data_task(self, latitude: float, longitude: float):
"""
واکشی داده‌های خاک برای مختصات داده‌شده و ذخیره در DB.
برای هر عمق (0-5cm, 5-15cm, 15-30cm) یک ریکوئست/شبیه‌سازی جدا انجام می‌شود.
اجرای async تحلیل سنجش‌ازدور برای location/block و ذخیره نتایج در DB.
"""
logger.info(
"Starting remote sensing analysis task",
extra={
"task_id": self.request.id,
"soil_location_id": soil_location_id,
"block_code": block_code,
"temporal_start": temporal_start,
"temporal_end": temporal_end,
"force_refresh": force_refresh,
},
)
try:
return fetch_soil_data_for_coordinates(
latitude=latitude,
longitude=longitude,
return run_remote_sensing_analysis(
soil_location_id=soil_location_id,
block_code=block_code,
temporal_start=temporal_start,
temporal_end=temporal_end,
force_refresh=force_refresh,
task_id=self.request.id,
progress_callback=self.update_state,
run_id=run_id,
cluster_count=cluster_count,
selected_features=selected_features,
)
except RequestException as exc:
lat = Decimal(str(round(float(latitude), 6)))
lon = Decimal(str(round(float(longitude), 6)))
location = SoilLocation.objects.filter(latitude=lat, longitude=lon).first()
return {
"status": "error",
"location_id": getattr(location, "id", None),
"error": str(exc),
}
except OpenEOAuthenticationError:
logger.exception(
"Remote sensing auth failure",
extra={"task_id": self.request.id, "soil_location_id": soil_location_id},
)
raise
except (OpenEOExecutionError, OpenEOServiceError, RequestException, DataDrivenSubdivisionError) as exc:
logger.warning(
"Transient remote sensing failure, retrying task",
extra={
"task_id": self.request.id,
"soil_location_id": soil_location_id,
"block_code": block_code,
"retry_count": self.request.retries,
"error": str(exc),
},
)
raise self.retry(exc=exc)
def _normalize_temporal_date(value: Any, field_name: str):
if hasattr(value, "isoformat") and not isinstance(value, str):
return value
parsed = parse_date(str(value))
if parsed is None:
raise ValueError(f"{field_name} نامعتبر است.")
return parsed
def _resolve_block_subdivision(location: SoilLocation, block_code: str) -> BlockSubdivision | None:
if not block_code:
return None
return (
BlockSubdivision.objects.filter(
soil_location=location,
block_code=block_code,
)
.order_by("-updated_at", "-id")
.first()
)
def _get_or_create_remote_sensing_run(
*,
run_id: int | None,
location: SoilLocation,
subdivision: BlockSubdivision | None,
block_code: str,
temporal_start,
temporal_end,
task_id: str,
cluster_count: int | None,
selected_features: list[str],
) -> RemoteSensingRun:
queued_at = timezone.now().isoformat()
if run_id is not None:
run = RemoteSensingRun.objects.filter(pk=run_id, soil_location=location).first()
if run is not None:
metadata = dict(run.metadata or {})
if task_id:
metadata["task_id"] = task_id
metadata.setdefault("status_label", "pending")
metadata["stage"] = "queued"
metadata["selected_features"] = selected_features
metadata["requested_cluster_count"] = cluster_count
metadata["pipeline"] = {
"name": "remote_sensing_subdivision",
"version": 2,
}
metadata["timestamps"] = {
**dict(metadata.get("timestamps") or {}),
"queued_at": queued_at,
}
run.block_subdivision = subdivision
run.block_code = block_code
run.chunk_size_sqm = int(getattr(settings, "SUBDIVISION_CHUNK_SQM", 900) or 900)
run.temporal_start = temporal_start
run.temporal_end = temporal_end
run.metadata = metadata
run.save(
update_fields=[
"block_subdivision",
"block_code",
"chunk_size_sqm",
"temporal_start",
"temporal_end",
"metadata",
"updated_at",
]
)
return run
metadata = {
"status_label": "pending",
"stage": "queued",
"selected_features": selected_features,
"requested_cluster_count": cluster_count,
"pipeline": {
"name": "remote_sensing_subdivision",
"version": 2,
},
"timestamps": {"queued_at": queued_at},
}
if task_id:
metadata["task_id"] = task_id
return RemoteSensingRun.objects.create(
soil_location=location,
block_subdivision=subdivision,
block_code=block_code,
chunk_size_sqm=int(getattr(settings, "SUBDIVISION_CHUNK_SQM", 900) or 900),
temporal_start=temporal_start,
temporal_end=temporal_end,
status=RemoteSensingRun.STATUS_PENDING,
metadata=metadata,
)
def _mark_run_running(run: RemoteSensingRun) -> None:
metadata = dict(run.metadata or {})
metadata["status_label"] = "running"
metadata["stage"] = "running"
metadata["timestamps"] = {
**dict(metadata.get("timestamps") or {}),
"started_at": timezone.now().isoformat(),
}
run.status = RemoteSensingRun.STATUS_RUNNING
run.started_at = timezone.now()
run.metadata = metadata
run.save(update_fields=["status", "started_at", "metadata", "updated_at"])
def _mark_run_success(
run: RemoteSensingRun,
summary: dict[str, Any],
service_metadata: dict[str, Any] | None = None,
) -> None:
metadata = dict(run.metadata or {})
metadata["summary"] = summary
metadata["status_label"] = "completed"
metadata["stage"] = "completed"
metadata["timestamps"] = {
**dict(metadata.get("timestamps") or {}),
"completed_at": timezone.now().isoformat(),
}
if service_metadata:
metadata["service"] = service_metadata
run.status = RemoteSensingRun.STATUS_SUCCESS
run.finished_at = timezone.now()
run.error_message = ""
run.metadata = metadata
run.save(
update_fields=[
"status",
"finished_at",
"error_message",
"metadata",
"updated_at",
]
)
def _mark_run_failure(run: RemoteSensingRun, error_message: str) -> None:
metadata = dict(run.metadata or {})
metadata["status_label"] = "failed"
metadata["failure_reason"] = error_message[:4000]
metadata["timestamps"] = {
**dict(metadata.get("timestamps") or {}),
"failed_at": timezone.now().isoformat(),
}
run.status = RemoteSensingRun.STATUS_FAILURE
run.finished_at = timezone.now()
run.error_message = error_message[:4000]
run.metadata = metadata
run.save(
update_fields=[
"status",
"finished_at",
"error_message",
"metadata",
"updated_at",
]
)
logger.exception(
"Remote sensing analysis failed",
extra={"run_id": run.id, "soil_location_id": run.soil_location_id, "block_code": run.block_code},
)
def _load_grid_cells(location: SoilLocation, block_code: str) -> list[AnalysisGridCell]:
queryset = AnalysisGridCell.objects.filter(soil_location=location)
queryset = queryset.filter(block_code=block_code or "")
return list(queryset.order_by("cell_code"))
def _load_observations(
*,
location: SoilLocation,
block_code: str,
temporal_start,
temporal_end,
) -> list[AnalysisGridObservation]:
queryset = (
AnalysisGridObservation.objects.select_related("cell", "run")
.filter(
cell__soil_location=location,
cell__block_code=block_code or "",
temporal_start=temporal_start,
temporal_end=temporal_end,
)
.order_by("cell__cell_code")
)
return list(queryset)
def _select_cells_for_processing(
*,
all_cells: list[AnalysisGridCell],
temporal_start,
temporal_end,
force_refresh: bool,
) -> list[AnalysisGridCell]:
if force_refresh:
return all_cells
existing_ids = set(
AnalysisGridObservation.objects.filter(
cell__in=all_cells,
temporal_start=temporal_start,
temporal_end=temporal_end,
).values_list("cell_id", flat=True)
)
return [cell for cell in all_cells if cell.id not in existing_ids]
def _upsert_grid_observations(
*,
cells: list[AnalysisGridCell],
run: RemoteSensingRun,
temporal_start,
temporal_end,
metric_payload: dict[str, Any],
) -> dict[str, int]:
metadata_template = {
"backend_name": metric_payload["metadata"].get("backend"),
"backend_url": metric_payload["metadata"].get("backend_url"),
"collections_used": metric_payload["metadata"].get("collections_used", []),
"slope_supported": metric_payload["metadata"].get("slope_supported", False),
"job_refs": metric_payload["metadata"].get("job_refs", {}),
"failed_metrics": metric_payload["metadata"].get("failed_metrics", []),
"run_id": run.id,
}
result_by_cell = metric_payload.get("results", {})
created_count = 0
updated_count = 0
with transaction.atomic():
for cell in cells:
values = result_by_cell.get(cell.cell_code, {})
defaults = {
"run": run,
"ndvi": values.get("ndvi"),
"ndwi": values.get("ndwi"),
"lst_c": values.get("lst_c"),
"soil_vv": values.get("soil_vv"),
"soil_vv_db": values.get("soil_vv_db"),
"dem_m": values.get("dem_m"),
"slope_deg": values.get("slope_deg"),
"metadata": metadata_template,
}
observation, created = AnalysisGridObservation.objects.update_or_create(
cell=cell,
temporal_start=temporal_start,
temporal_end=temporal_end,
defaults=defaults,
)
if created:
created_count += 1
else:
updated_count += 1
return {"created_count": created_count, "updated_count": updated_count}
def _ensure_subdivision_result(
*,
location: SoilLocation,
run: RemoteSensingRun,
subdivision: BlockSubdivision | None,
block_code: str,
observations: list[AnalysisGridObservation],
cluster_count: int | None,
selected_features: list[str] | None,
) -> RemoteSensingSubdivisionResult:
if not observations:
raise DataDrivenSubdivisionError("هیچ observation برای ساخت subdivision داده‌محور پیدا نشد.")
result = create_remote_sensing_subdivision_result(
location=location,
run=run,
observations=observations,
block_subdivision=subdivision,
block_code=block_code,
selected_features=selected_features or list(DEFAULT_CLUSTER_FEATURES),
explicit_k=cluster_count,
)
return result
def _record_run_stage(run: RemoteSensingRun, stage: str, details: dict[str, Any] | None = None) -> None:
metadata = dict(run.metadata or {})
metadata["stage"] = stage
metadata["stage_details"] = {
**dict(metadata.get("stage_details") or {}),
stage: details or {},
}
metadata["timestamps"] = {
**dict(metadata.get("timestamps") or {}),
f"{stage}_at": timezone.now().isoformat(),
}
run.metadata = metadata
run.save(update_fields=["metadata", "updated_at"])
def _build_clustering_stage_metadata(
result: RemoteSensingSubdivisionResult,
) -> dict[str, Any]:
metadata = dict(result.metadata or {})
return {
"subdivision_result_id": result.id,
"cluster_count": result.cluster_count,
"selected_features": result.selected_features,
"used_cell_count": metadata.get("used_cell_count", 0),
"skipped_cell_count": metadata.get("skipped_cell_count", 0),
"skipped_cell_codes": result.skipped_cell_codes,
"kmeans_params": metadata.get("kmeans_params", {}),
}
+44
View File
@@ -0,0 +1,44 @@
from django.test import SimpleTestCase, override_settings
from location_data.block_subdivision import (
build_block_subdivision_payload,
detect_elbow_point,
)
@override_settings(SUBDIVISION_CHUNK_SQM=900)
class BlockSubdivisionServiceTests(SimpleTestCase):
def test_detect_elbow_point_from_sse_curve(self):
inertia_curve = [
{"k": 1, "sse": 1000.0},
{"k": 2, "sse": 400.0},
{"k": 3, "sse": 220.0},
{"k": 4, "sse": 180.0},
]
optimal_k = detect_elbow_point(inertia_curve)
self.assertEqual(optimal_k, 2)
def test_build_block_subdivision_payload_returns_grid_and_centroids(self):
boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3890, 35.6890],
[51.3902, 35.6890],
[51.3902, 35.6900],
[51.3890, 35.6900],
[51.3890, 35.6890],
]
],
}
result = build_block_subdivision_payload(boundary, block_code="block-1")
self.assertEqual(result["block_code"], "block-1")
self.assertEqual(result["chunk_size_sqm"], 900)
self.assertGreater(result["grid_point_count"], 0)
self.assertGreater(result["centroid_count"], 0)
self.assertIn("optimal_k", result["metadata"])
self.assertTrue(result["metadata"]["inertia_curve"])
@@ -0,0 +1,135 @@
from datetime import date
from django.test import TestCase
from location_data.data_driven_subdivision import sync_block_subdivision_with_result
from location_data.models import (
AnalysisGridCell,
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingRun,
RemoteSensingSubdivisionResult,
SoilLocation,
)
class DataDrivenSubdivisionSyncTests(TestCase):
def setUp(self):
self.boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3890, 35.6890],
[51.3900, 35.6890],
[51.3900, 35.6900],
[51.3890, 35.6900],
[51.3890, 35.6890],
]
],
}
self.location = SoilLocation.objects.create(
latitude="35.689200",
longitude="51.389000",
farm_boundary=self.boundary,
)
self.subdivision = BlockSubdivision.objects.create(
soil_location=self.location,
block_code="block-1",
source_boundary=self.boundary,
chunk_size_sqm=900,
status="defined",
)
self.run = RemoteSensingRun.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
status=RemoteSensingRun.STATUS_SUCCESS,
)
def test_sync_block_subdivision_with_result_updates_saved_sub_blocks(self):
cell_1 = AnalysisGridCell.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
cell_code="cell-1",
chunk_size_sqm=900,
geometry=self.boundary,
centroid_lat="35.689200",
centroid_lon="51.389200",
)
cell_2 = AnalysisGridCell.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
cell_code="cell-2",
chunk_size_sqm=900,
geometry=self.boundary,
centroid_lat="35.689700",
centroid_lon="51.389700",
)
observation_1 = AnalysisGridObservation.objects.create(
cell=cell_1,
run=self.run,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
ndvi=0.5,
)
observation_2 = AnalysisGridObservation.objects.create(
cell=cell_2,
run=self.run,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
ndvi=0.7,
)
result = RemoteSensingSubdivisionResult.objects.create(
soil_location=self.location,
run=self.run,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
cluster_count=2,
selected_features=["ndvi"],
metadata={
"used_cell_count": 2,
"skipped_cell_count": 0,
"inertia_curve": [{"k": 1, "sse": 1.0}, {"k": 2, "sse": 0.1}],
},
)
sync_block_subdivision_with_result(
block_subdivision=self.subdivision,
result=result,
observations=[observation_1, observation_2],
cluster_summaries=[
{
"cluster_label": 0,
"centroid_lat": 35.6892,
"centroid_lon": 51.3892,
"cell_count": 1,
"cell_codes": ["cell-1"],
},
{
"cluster_label": 1,
"centroid_lat": 35.6897,
"centroid_lon": 51.3897,
"cell_count": 1,
"cell_codes": ["cell-2"],
},
],
)
self.subdivision.refresh_from_db()
self.assertEqual(self.subdivision.status, "subdivided")
self.assertEqual(self.subdivision.grid_point_count, 2)
self.assertEqual(self.subdivision.centroid_count, 2)
self.assertEqual(self.subdivision.grid_points[0]["cell_code"], "cell-1")
self.assertEqual(self.subdivision.centroid_points[0]["sub_block_code"], "cluster-0")
self.assertEqual(
self.subdivision.metadata["data_driven_subdivision"]["cluster_count"],
2,
)
+114
View File
@@ -0,0 +1,114 @@
from django.test import TestCase, override_settings
from location_data.grid_analysis import create_or_get_analysis_grid_cells
from location_data.models import AnalysisGridCell, BlockSubdivision, SoilLocation
@override_settings(SUBDIVISION_CHUNK_SQM=900)
class AnalysisGridServiceTests(TestCase):
def setUp(self):
self.boundary = {
"type": "Polygon",
"coordinates": [
[
[51.389000, 35.689000],
[51.389760, 35.689000],
[51.389760, 35.689620],
[51.389000, 35.689620],
[51.389000, 35.689000],
]
],
}
self.location = SoilLocation.objects.create(
latitude="35.689310",
longitude="51.389380",
farm_boundary=self.boundary,
)
self.location.set_input_block_count(1)
self.location.save(update_fields=["input_block_count", "block_layout", "updated_at"])
self.subdivision = BlockSubdivision.objects.create(
soil_location=self.location,
block_code="block-1",
source_boundary=self.boundary,
chunk_size_sqm=900,
status="created",
)
def test_create_analysis_grid_cells_persists_30x30_cells(self):
result = create_or_get_analysis_grid_cells(
self.location,
block_code="block-1",
block_subdivision=self.subdivision,
)
self.assertTrue(result["created"])
self.assertEqual(result["chunk_size_sqm"], 900)
self.assertGreater(result["created_count"], 0)
self.assertEqual(result["created_count"], result["total_count"])
cells = list(
AnalysisGridCell.objects.filter(
soil_location=self.location,
block_code="block-1",
chunk_size_sqm=900,
).order_by("cell_code")
)
self.assertEqual(len(cells), result["total_count"])
self.assertTrue(all(cell.block_subdivision_id == self.subdivision.id for cell in cells))
self.assertTrue(all(cell.geometry.get("type") == "Polygon" for cell in cells))
self.assertTrue(all(len(cell.geometry.get("coordinates", [[]])[0]) == 5 for cell in cells))
self.subdivision.refresh_from_db()
self.location.refresh_from_db()
self.assertEqual(
self.subdivision.metadata["analysis_grid"]["chunk_size_sqm"],
900,
)
self.assertEqual(
self.subdivision.metadata["analysis_grid"]["cell_count"],
result["total_count"],
)
self.assertEqual(
self.location.block_layout["blocks"][0]["analysis_grid_summary"]["chunk_size_sqm"],
900,
)
def test_create_analysis_grid_cells_is_idempotent(self):
first = create_or_get_analysis_grid_cells(
self.location,
block_code="block-1",
block_subdivision=self.subdivision,
)
second = create_or_get_analysis_grid_cells(
self.location,
block_code="block-1",
block_subdivision=self.subdivision,
)
self.assertTrue(first["created"])
self.assertFalse(second["created"])
self.assertEqual(second["created_count"], 0)
self.assertEqual(second["existing_count"], first["total_count"])
self.assertEqual(
AnalysisGridCell.objects.filter(
soil_location=self.location,
block_code="block-1",
chunk_size_sqm=900,
).count(),
first["total_count"],
)
def test_create_analysis_grid_cells_uses_location_boundary_without_subdivision(self):
result = create_or_get_analysis_grid_cells(
self.location,
block_code="",
)
self.assertGreater(result["total_count"], 0)
self.assertTrue(
AnalysisGridCell.objects.filter(
soil_location=self.location,
block_code="",
chunk_size_sqm=900,
).exists()
)
+66
View File
@@ -0,0 +1,66 @@
from decimal import Decimal
from django.test import SimpleTestCase
from location_data.openeo_service import (
build_empty_metric_payload,
linear_to_db,
merge_metric_results,
parse_aggregate_spatial_response,
)
class OpenEOServiceParsingTests(SimpleTestCase):
def test_parse_feature_collection_results(self):
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "cell-1",
"properties": {"mean": 0.61},
},
{
"type": "Feature",
"id": "cell-2",
"properties": {"mean": 0.47},
},
],
}
result = parse_aggregate_spatial_response(payload, "ndvi")
self.assertEqual(result["cell-1"]["ndvi"], 0.61)
self.assertEqual(result["cell-2"]["ndvi"], 0.47)
def test_parse_mapping_results(self):
payload = {
"cell-1": {"mean": 12.4},
"cell-2": {"mean": 15.1},
}
result = parse_aggregate_spatial_response(payload, "lst_c")
self.assertEqual(result["cell-1"]["lst_c"], 12.4)
self.assertEqual(result["cell-2"]["lst_c"], 15.1)
def test_linear_to_db(self):
self.assertEqual(linear_to_db(10.0), 10.0)
self.assertEqual(linear_to_db(Decimal("1.0")), 0.0)
self.assertIsNone(linear_to_db(0))
self.assertIsNone(linear_to_db(-1))
def test_merge_metric_results(self):
target = {"cell-1": build_empty_metric_payload()}
merge_metric_results(
target,
{
"cell-1": {"ndvi": 0.5},
"cell-2": {"ndwi": 0.2},
},
)
self.assertEqual(target["cell-1"]["ndvi"], 0.5)
self.assertEqual(target["cell-2"]["ndwi"], 0.2)
self.assertIn("soil_vv_db", target["cell-2"])
+265
View File
@@ -0,0 +1,265 @@
from datetime import date
from types import SimpleNamespace
from unittest.mock import patch
from django.test import TestCase, override_settings
from rest_framework.test import APIClient
from location_data.models import (
AnalysisGridCell,
AnalysisGridObservation,
BlockSubdivision,
RemoteSensingClusterAssignment,
RemoteSensingRun,
RemoteSensingSubdivisionResult,
SoilLocation,
)
@override_settings(ROOT_URLCONF="location_data.urls")
class RemoteSensingApiTests(TestCase):
def setUp(self):
self.client = APIClient()
self.boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3890, 35.6890],
[51.3900, 35.6890],
[51.3900, 35.6900],
[51.3890, 35.6900],
[51.3890, 35.6890],
]
],
}
self.location = SoilLocation.objects.create(
latitude="35.689200",
longitude="51.389000",
farm_boundary=self.boundary,
)
self.location.set_input_block_count(1)
self.location.save(update_fields=["input_block_count", "block_layout", "updated_at"])
self.subdivision = BlockSubdivision.objects.create(
soil_location=self.location,
block_code="block-1",
source_boundary=self.boundary,
chunk_size_sqm=900,
status="created",
)
def test_post_remote_sensing_returns_404_when_location_missing(self):
response = self.client.post(
"/remote-sensing/",
data={
"lat": 35.7000,
"lon": 51.4000,
"start_date": "2025-01-01",
"end_date": "2025-01-31",
},
format="json",
)
self.assertEqual(response.status_code, 404)
self.assertEqual(response.json()["msg"], "location پیدا نشد.")
@patch("location_data.views.run_remote_sensing_analysis_task.delay")
def test_post_remote_sensing_enqueues_task_and_returns_processing(self, mock_delay):
mock_delay.return_value = SimpleNamespace(id="task-123")
response = self.client.post(
"/remote-sensing/",
data={
"lat": 35.6892,
"lon": 51.3890,
"block_code": "block-1",
"start_date": "2025-01-01",
"end_date": "2025-01-31",
"force_refresh": False,
},
format="json",
)
self.assertEqual(response.status_code, 202)
payload = response.json()["data"]
self.assertEqual(payload["status"], "processing")
self.assertEqual(payload["source"], "processing")
self.assertEqual(payload["task_id"], "task-123")
self.assertEqual(payload["block_code"], "block-1")
self.assertEqual(payload["summary"]["cell_count"], 0)
run = RemoteSensingRun.objects.get(id=payload["run"]["id"])
self.assertEqual(run.block_code, "block-1")
self.assertEqual(run.status, RemoteSensingRun.STATUS_PENDING)
self.assertEqual(run.metadata["stage"], "queued")
self.assertEqual(run.metadata["selected_features"], [])
mock_delay.assert_called_once()
def test_get_remote_sensing_returns_processing_when_run_exists_without_results(self):
RemoteSensingRun.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
status=RemoteSensingRun.STATUS_RUNNING,
metadata={"task_id": "task-123"},
)
response = self.client.get(
"/remote-sensing/",
data={
"lat": 35.6892,
"lon": 51.3890,
"block_code": "block-1",
"start_date": "2025-01-01",
"end_date": "2025-01-31",
},
)
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["status"], "processing")
self.assertEqual(payload["source"], "processing")
self.assertEqual(payload["cells"], [])
self.assertEqual(payload["run"]["status"], RemoteSensingRun.STATUS_RUNNING)
def test_get_remote_sensing_returns_cached_results(self):
run = RemoteSensingRun.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
status=RemoteSensingRun.STATUS_SUCCESS,
)
cell = AnalysisGridCell.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
cell_code="cell-1",
chunk_size_sqm=900,
geometry=self.boundary,
centroid_lat="35.689500",
centroid_lon="51.389500",
)
AnalysisGridObservation.objects.create(
cell=cell,
run=run,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
ndvi=0.61,
ndwi=0.22,
lst_c=24.5,
soil_vv=0.13,
soil_vv_db=-8.860566,
dem_m=1550.0,
slope_deg=4.2,
metadata={"backend_name": "openeo"},
)
response = self.client.get(
"/remote-sensing/",
data={
"lat": 35.6892,
"lon": 51.3890,
"block_code": "block-1",
"start_date": "2025-01-01",
"end_date": "2025-01-31",
},
)
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["status"], "success")
self.assertEqual(payload["source"], "database")
self.assertEqual(payload["summary"]["cell_count"], 1)
self.assertEqual(payload["summary"]["ndvi_mean"], 0.61)
self.assertEqual(payload["summary"]["soil_vv_db_mean"], -8.860566)
self.assertEqual(len(payload["cells"]), 1)
self.assertEqual(payload["cells"][0]["cell_code"], "cell-1")
def test_run_status_endpoint_returns_normalized_status(self):
run = RemoteSensingRun.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
status=RemoteSensingRun.STATUS_SUCCESS,
metadata={"stage": "completed", "selected_features": ["ndvi"]},
)
response = self.client.get(f"/remote-sensing/runs/{run.id}/status/")
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["status"], "completed")
self.assertEqual(payload["run"]["pipeline_status"], "completed")
self.assertEqual(payload["run"]["stage"], "completed")
self.assertEqual(payload["run"]["selected_features"], ["ndvi"])
def test_run_result_endpoint_returns_paginated_assignments(self):
run = RemoteSensingRun.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
status=RemoteSensingRun.STATUS_SUCCESS,
metadata={"stage": "completed"},
)
cell = AnalysisGridCell.objects.create(
soil_location=self.location,
block_subdivision=self.subdivision,
block_code="block-1",
cell_code="cell-1",
chunk_size_sqm=900,
geometry=self.boundary,
centroid_lat="35.689500",
centroid_lon="51.389500",
)
AnalysisGridObservation.objects.create(
cell=cell,
run=run,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
ndvi=0.61,
ndwi=0.22,
lst_c=24.5,
soil_vv=0.13,
soil_vv_db=-8.860566,
dem_m=1550.0,
slope_deg=4.2,
metadata={"backend_name": "openeo"},
)
result = RemoteSensingSubdivisionResult.objects.create(
soil_location=self.location,
run=run,
block_subdivision=self.subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start=date(2025, 1, 1),
temporal_end=date(2025, 1, 31),
cluster_count=1,
selected_features=["ndvi"],
metadata={"used_cell_count": 1, "skipped_cell_count": 0},
)
RemoteSensingClusterAssignment.objects.create(
result=result,
cell=cell,
cluster_label=0,
raw_feature_values={"ndvi": 0.61},
scaled_feature_values={"ndvi": 0.0},
)
response = self.client.get(f"/remote-sensing/runs/{run.id}/result/", data={"page": 1, "page_size": 10})
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["status"], "completed")
self.assertEqual(payload["subdivision_result"]["cluster_count"], 1)
self.assertEqual(len(payload["subdivision_result"]["assignments"]), 1)
self.assertEqual(payload["pagination"]["assignments"]["total_items"], 1)
-92
View File
@@ -1,92 +0,0 @@
from __future__ import annotations
from django.apps import apps
from django.test import SimpleTestCase, TestCase, override_settings
from location_data.models import SoilDepthData, SoilLocation
from location_data.soil_adapters import (
DEPTHS,
MockSoilDataAdapter,
SoilGridsAdapter,
)
from location_data.tasks import fetch_soil_data_for_coordinates
class MockSoilDataAdapterTests(SimpleTestCase):
def setUp(self):
self.adapter = MockSoilDataAdapter(delay_seconds=0)
def test_same_coordinate_returns_same_values(self):
first = self.adapter.fetch_depth_fields(51.4, 35.71, "0-5cm")
second = self.adapter.fetch_depth_fields(51.4, 35.71, "0-5cm")
self.assertEqual(first, second)
def test_nearby_coordinates_produce_nearby_values(self):
first = self.adapter.fetch_depth_fields(51.4, 35.71, "0-5cm")
second = self.adapter.fetch_depth_fields(51.405, 35.715, "0-5cm")
self.assertLess(abs(first["sand"] - second["sand"]), 4.5)
self.assertLess(abs(first["clay"] - second["clay"]), 4.5)
self.assertLess(abs(first["phh2o"] - second["phh2o"]), 0.35)
self.assertLess(abs(first["wv1500"] - second["wv1500"]), 0.03)
def test_depth_profiles_follow_expected_trend(self):
shallow = self.adapter.fetch_depth_fields(51.4, 35.71, "0-5cm")
medium = self.adapter.fetch_depth_fields(51.4, 35.71, "5-15cm")
deep = self.adapter.fetch_depth_fields(51.4, 35.71, "15-30cm")
self.assertGreaterEqual(deep["bdod"], medium["bdod"])
self.assertGreaterEqual(medium["bdod"], shallow["bdod"])
self.assertLessEqual(deep["soc"], medium["soc"])
self.assertLessEqual(medium["soc"], shallow["soc"])
class SoilDataAdapterSelectionTests(SimpleTestCase):
def tearDown(self):
apps.get_app_config("location_data").__dict__.pop("soil_data_adapter", None)
@override_settings(SOIL_DATA_PROVIDER="mock", SOIL_MOCK_DELAY_SECONDS=0)
def test_app_config_returns_mock_adapter(self):
config = apps.get_app_config("location_data")
config.__dict__.pop("soil_data_adapter", None)
adapter = config.get_soil_data_adapter()
self.assertIsInstance(adapter, MockSoilDataAdapter)
@override_settings(SOIL_DATA_PROVIDER="soilgrids", SOILGRIDS_TIMEOUT_SECONDS=12)
def test_app_config_returns_live_adapter(self):
config = apps.get_app_config("location_data")
config.__dict__.pop("soil_data_adapter", None)
adapter = config.get_soil_data_adapter()
self.assertIsInstance(adapter, SoilGridsAdapter)
self.assertEqual(adapter.timeout, 12)
@override_settings(SOIL_DATA_PROVIDER="mock", SOIL_MOCK_DELAY_SECONDS=0)
class SoilDataFetchTests(TestCase):
def test_fetch_soil_data_for_coordinates_persists_three_depths(self):
result = fetch_soil_data_for_coordinates(latitude=35.71, longitude=51.4)
self.assertEqual(result["status"], "completed")
self.assertEqual(result["depths"], DEPTHS)
location = SoilLocation.objects.get(latitude="35.710000", longitude="51.400000")
self.assertEqual(location.depths.count(), 3)
self.assertTrue(location.is_complete)
self.assertCountEqual(
list(location.depths.values_list("depth_label", flat=True)),
DEPTHS,
)
self.assertTrue(
SoilDepthData.objects.filter(
soil_location=location,
depth_label="0-5cm",
sand__isnull=False,
clay__isnull=False,
wv1500__isnull=False,
).exists()
)
+257
View File
@@ -0,0 +1,257 @@
from django.test import TestCase, override_settings
from rest_framework.test import APIClient
from location_data.models import AnalysisGridCell, BlockSubdivision, RemoteSensingRun, SoilLocation
@override_settings(ROOT_URLCONF="location_data.urls")
class SoilDataApiTests(TestCase):
def setUp(self):
self.client = APIClient()
self.boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3890, 35.6890],
[51.3902, 35.6890],
[51.3902, 35.6900],
[51.3890, 35.6900],
[51.3890, 35.6890],
]
],
}
self.block_boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3890, 35.6890],
[51.3896, 35.6890],
[51.3896, 35.6900],
[51.3890, 35.6900],
[51.3890, 35.6890],
]
],
}
def test_post_creates_default_single_block_layout(self):
response = self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{
"block_code": "block-1",
"boundary": self.block_boundary,
}
],
},
format="json",
)
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["source"], "created")
self.assertEqual(payload["input_block_count"], 1)
self.assertEqual(len(payload["block_layout"]["blocks"]), 1)
self.assertEqual(payload["block_layout"]["blocks"][0]["boundary"], self.block_boundary)
self.assertEqual(payload["block_layout"]["algorithm_status"], "pending")
self.assertEqual(len(payload["block_subdivisions"]), 1)
self.assertEqual(payload["block_subdivisions"][0]["status"], "defined")
self.assertEqual(payload["satellite_snapshots"][0]["status"], "missing")
def test_post_updates_block_layout_from_input(self):
SoilLocation.objects.create(
latitude="35.689200",
longitude="51.389000",
)
response = self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{"block_code": "block-a", "boundary": self.block_boundary},
{"block_code": "block-b", "boundary": self.block_boundary},
],
},
format="json",
)
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(payload["input_block_count"], 2)
self.assertEqual(len(payload["block_layout"]["blocks"]), 2)
self.assertEqual(len(payload["block_subdivisions"]), 2)
location = SoilLocation.objects.get(latitude="35.689200", longitude="51.389000")
self.assertEqual(location.input_block_count, 2)
self.assertEqual(len(location.block_layout["blocks"]), 2)
self.assertEqual(location.block_layout["algorithm_status"], "pending")
self.assertTrue(
BlockSubdivision.objects.filter(
soil_location=location,
block_code="block-a",
status="defined",
).exists()
)
def test_get_returns_stored_subdivisions_without_processing(self):
self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{
"block_code": "block-1",
"boundary": self.block_boundary,
}
],
},
format="json",
)
response = self.client.get(
"/",
data={"lat": 35.6892, "lon": 51.3890},
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["data"]["source"], "database")
self.assertEqual(len(response.json()["data"]["block_subdivisions"]), 1)
def test_post_removes_blocks_not_present_in_latest_farmer_input(self):
self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{"block_code": "block-a", "boundary": self.block_boundary},
{"block_code": "block-b", "boundary": self.block_boundary},
],
},
format="json",
)
response = self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{"block_code": "block-a", "boundary": self.block_boundary},
],
},
format="json",
)
self.assertEqual(response.status_code, 200)
payload = response.json()["data"]
self.assertEqual(len(payload["block_subdivisions"]), 1)
self.assertEqual(payload["block_subdivisions"][0]["block_code"], "block-a")
location = SoilLocation.objects.get(latitude="35.689200", longitude="51.389000")
self.assertTrue(
BlockSubdivision.objects.filter(soil_location=location, block_code="block-a").exists()
)
self.assertFalse(
BlockSubdivision.objects.filter(soil_location=location, block_code="block-b").exists()
)
def test_post_clears_cached_grid_and_run_when_block_boundary_changes(self):
self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{"block_code": "block-1", "boundary": self.block_boundary},
],
},
format="json",
)
location = SoilLocation.objects.get(latitude="35.689200", longitude="51.389000")
subdivision = BlockSubdivision.objects.get(soil_location=location, block_code="block-1")
AnalysisGridCell.objects.create(
soil_location=location,
block_subdivision=subdivision,
block_code="block-1",
cell_code="cell-1",
chunk_size_sqm=900,
geometry=self.block_boundary,
centroid_lat="35.689500",
centroid_lon="51.389300",
)
RemoteSensingRun.objects.create(
soil_location=location,
block_subdivision=subdivision,
block_code="block-1",
chunk_size_sqm=900,
temporal_start="2025-01-01",
temporal_end="2025-01-31",
status=RemoteSensingRun.STATUS_SUCCESS,
)
subdivision.grid_points = [{"cell_code": "cell-1"}]
subdivision.centroid_points = [{"sub_block_code": "cluster-0"}]
subdivision.grid_point_count = 1
subdivision.centroid_count = 1
subdivision.status = "subdivided"
subdivision.save(
update_fields=[
"grid_points",
"centroid_points",
"grid_point_count",
"centroid_count",
"status",
"updated_at",
]
)
updated_boundary = {
"type": "Polygon",
"coordinates": [
[
[51.3892, 35.6890],
[51.3898, 35.6890],
[51.3898, 35.6900],
[51.3892, 35.6900],
[51.3892, 35.6890],
]
],
}
response = self.client.post(
"/",
data={
"lat": 35.6892,
"lon": 51.3890,
"farm_boundary": self.boundary,
"blocks": [
{"block_code": "block-1", "boundary": updated_boundary},
],
},
format="json",
)
self.assertEqual(response.status_code, 200)
subdivision.refresh_from_db()
self.assertEqual(subdivision.status, "defined")
self.assertEqual(subdivision.source_boundary, updated_boundary)
self.assertEqual(subdivision.grid_points, [])
self.assertEqual(subdivision.centroid_points, [])
self.assertEqual(subdivision.grid_point_count, 0)
self.assertEqual(subdivision.centroid_count, 0)
self.assertFalse(
AnalysisGridCell.objects.filter(soil_location=location, block_code="block-1").exists()
)
self.assertFalse(
RemoteSensingRun.objects.filter(soil_location=location, block_code="block-1").exists()
)
+10 -2
View File
@@ -1,9 +1,17 @@
from django.urls import path
from .views import NdviHealthView, SoilDataTaskStatusView, SoilDataView
from .views import (
NdviHealthView,
RemoteSensingAnalysisView,
RemoteSensingRunResultView,
RemoteSensingRunStatusView,
SoilDataView,
)
urlpatterns = [
path("", SoilDataView.as_view(), name="soil-data"),
path("remote-sensing/", RemoteSensingAnalysisView.as_view(), name="remote-sensing"),
path("remote-sensing/runs/<int:run_id>/status/", RemoteSensingRunStatusView.as_view(), name="remote-sensing-run-status"),
path("remote-sensing/runs/<int:run_id>/result/", RemoteSensingRunResultView.as_view(), name="remote-sensing-run-result"),
path("ndvi-health/", NdviHealthView.as_view(), name="ndvi-health"),
path("tasks/<str:task_id>/status/", SoilDataTaskStatusView.as_view(), name="soil-data-task-status"),
]
+860 -117
View File
File diff suppressed because it is too large Load Diff