404 lines
15 KiB
Python
404 lines
15 KiB
Python
from decimal import Decimal
|
|
from io import StringIO
|
|
import os
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from unittest.mock import Mock, patch
|
|
|
|
from django.core.management import call_command
|
|
from django.test import SimpleTestCase
|
|
import requests
|
|
|
|
from config.proxy import resolve_requests_proxy_url
|
|
from location_data.openeo_service import (
|
|
OpenEOConnectionSettings,
|
|
OpenEOServiceError,
|
|
OpenEOExecutionError,
|
|
_log_raw_payload_summary,
|
|
_load_first_json_payload,
|
|
_load_job_result_payload,
|
|
_resolve_openeo_proxy_url_from_env,
|
|
_run_aggregate_spatial_job,
|
|
log_openeo_request_summary,
|
|
build_openeo_requests_session,
|
|
build_empty_metric_payload,
|
|
connect_openeo,
|
|
is_openeo_auth_configured,
|
|
linear_to_db,
|
|
merge_metric_results,
|
|
parse_aggregate_spatial_response,
|
|
)
|
|
|
|
|
|
class OpenEOServiceParsingTests(SimpleTestCase):
|
|
def test_parse_feature_collection_results(self):
|
|
payload = {
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"id": "cell-1",
|
|
"properties": {"mean": 0.61},
|
|
},
|
|
{
|
|
"type": "Feature",
|
|
"id": "cell-2",
|
|
"properties": {"mean": 0.47},
|
|
},
|
|
],
|
|
}
|
|
|
|
result = parse_aggregate_spatial_response(payload, "ndvi")
|
|
|
|
self.assertEqual(result["cell-1"]["ndvi"], 0.61)
|
|
self.assertEqual(result["cell-2"]["ndvi"], 0.47)
|
|
|
|
def test_parse_mapping_results(self):
|
|
payload = {
|
|
"cell-1": {"mean": 12.4},
|
|
"cell-2": {"mean": 15.1},
|
|
}
|
|
|
|
result = parse_aggregate_spatial_response(payload, "ndwi")
|
|
|
|
self.assertEqual(result["cell-1"]["ndwi"], 12.4)
|
|
self.assertEqual(result["cell-2"]["ndwi"], 15.1)
|
|
|
|
def test_parse_mapping_results_maps_numeric_keys_to_expected_feature_ids(self):
|
|
payload = {
|
|
"0": {"mean": 12.4},
|
|
"1": {"mean": 15.1},
|
|
}
|
|
|
|
result = parse_aggregate_spatial_response(
|
|
payload,
|
|
"ndwi",
|
|
expected_feature_ids=["cell-1", "cell-2"],
|
|
)
|
|
|
|
self.assertEqual(result["cell-1"]["ndwi"], 12.4)
|
|
self.assertEqual(result["cell-2"]["ndwi"], 15.1)
|
|
|
|
def test_parse_list_results_maps_positional_payload_to_expected_feature_ids(self):
|
|
payload = [{"mean": 0.61}, {"mean": 0.47}]
|
|
|
|
result = parse_aggregate_spatial_response(
|
|
payload,
|
|
"ndvi",
|
|
expected_feature_ids=["cell-1", "cell-2"],
|
|
)
|
|
|
|
self.assertEqual(result["cell-1"]["ndvi"], 0.61)
|
|
self.assertEqual(result["cell-2"]["ndvi"], 0.47)
|
|
|
|
def test_parse_list_results_extracts_scalar_from_nested_list_payloads(self):
|
|
payload = [[0.61], [0.47]]
|
|
|
|
result = parse_aggregate_spatial_response(
|
|
payload,
|
|
"ndvi",
|
|
expected_feature_ids=["cell-1", "cell-2"],
|
|
)
|
|
|
|
self.assertEqual(result["cell-1"]["ndvi"], 0.61)
|
|
self.assertEqual(result["cell-2"]["ndvi"], 0.47)
|
|
|
|
def test_log_raw_payload_summary_warns_for_empty_payload(self):
|
|
with self.assertLogs("location_data.openeo_service", level="WARNING") as captured:
|
|
summary = _log_raw_payload_summary({}, metric_name="ndvi", job_ref="job-1")
|
|
|
|
self.assertEqual(summary["returned_cell_count"], 0)
|
|
self.assertIn("openEO payload is empty for job_ref=job-1", "\n".join(captured.output))
|
|
|
|
def test_parse_logs_feature_mismatch_for_unexpected_keys(self):
|
|
payload = {"cell-1": {"foo": 12.4}}
|
|
|
|
with self.assertLogs("location_data.openeo_service", level="WARNING") as captured:
|
|
result = parse_aggregate_spatial_response(payload, "ndwi", job_ref="job-2")
|
|
|
|
self.assertEqual(result["cell-1"]["ndwi"], 12.4)
|
|
self.assertIn("Feature mismatch for cell=cell-1, available_keys=['foo']", "\n".join(captured.output))
|
|
|
|
def test_linear_to_db(self):
|
|
self.assertEqual(linear_to_db(10.0), 10.0)
|
|
self.assertEqual(linear_to_db(Decimal("1.0")), 0.0)
|
|
self.assertIsNone(linear_to_db(0))
|
|
self.assertIsNone(linear_to_db(-1))
|
|
|
|
def test_merge_metric_results(self):
|
|
target = {"cell-1": build_empty_metric_payload()}
|
|
|
|
merge_metric_results(
|
|
target,
|
|
{
|
|
"cell-1": {"ndvi": 0.5},
|
|
"cell-2": {"ndwi": 0.2},
|
|
},
|
|
)
|
|
|
|
self.assertEqual(target["cell-1"]["ndvi"], 0.5)
|
|
self.assertEqual(target["cell-2"]["ndwi"], 0.2)
|
|
self.assertIn("soil_vv_db", target["cell-2"])
|
|
|
|
def test_log_openeo_request_summary_logs_expected_fields(self):
|
|
cell = Mock()
|
|
cell.cell_code = "cell-1"
|
|
with self.assertLogs("location_data.openeo_service", level="INFO") as captured:
|
|
log_openeo_request_summary(
|
|
cells=[cell],
|
|
temporal_start="2026-04-08",
|
|
temporal_end="2026-05-08",
|
|
spatial_extent={"west": 49.9995, "south": 49.9995, "east": 50.0005, "north": 50.0005},
|
|
selected_features=["ndvi", "ndwi"],
|
|
)
|
|
|
|
joined = "\n".join(captured.output)
|
|
self.assertIn("openEO request summary", joined)
|
|
self.assertIn('"cell_count": 1', joined)
|
|
self.assertIn('"date_range_days": 31', joined)
|
|
self.assertIn('"metrics": ["ndvi", "ndwi"]', joined)
|
|
|
|
|
|
class OpenEOConnectionTests(SimpleTestCase):
|
|
def test_default_openeo_timeout_is_ten_minutes(self):
|
|
with patch.dict(os.environ, {}, clear=True):
|
|
settings = OpenEOConnectionSettings.from_env()
|
|
|
|
self.assertEqual(settings.timeout_seconds, 600.0)
|
|
|
|
def test_default_openeo_proxy_url_uses_proxychains_endpoint_without_wrapping_process(self):
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
"ENABLE_PROXYCHAINS": "0",
|
|
"PROXYCHAINS_PROXY_TYPE": "socks4",
|
|
"PROXYCHAINS_PROXY_HOST": "host.docker.internal",
|
|
"PROXYCHAINS_PROXY_PORT": "10808",
|
|
"OPENEO_PROXY_URL": "socks5h://host.docker.internal:10808",
|
|
},
|
|
clear=False,
|
|
):
|
|
self.assertEqual(
|
|
_resolve_openeo_proxy_url_from_env(),
|
|
"socks4a://host.docker.internal:10808",
|
|
)
|
|
|
|
def test_requests_proxy_is_disabled_when_proxychains_targets_same_endpoint(self):
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
"ENABLE_PROXYCHAINS": "1",
|
|
"PROXYCHAINS_PROXY_TYPE": "socks4",
|
|
"PROXYCHAINS_PROXY_HOST": "host.docker.internal",
|
|
"PROXYCHAINS_PROXY_PORT": "10808",
|
|
},
|
|
clear=False,
|
|
):
|
|
self.assertEqual(
|
|
resolve_requests_proxy_url("socks5h://host.docker.internal:10808"),
|
|
"",
|
|
)
|
|
|
|
def test_is_openeo_auth_configured_for_client_credentials(self):
|
|
self.assertTrue(
|
|
is_openeo_auth_configured(
|
|
OpenEOConnectionSettings(
|
|
auth_method="client_credentials",
|
|
client_id="client-id",
|
|
client_secret="client-secret",
|
|
)
|
|
)
|
|
)
|
|
|
|
def test_is_openeo_auth_configured_for_password(self):
|
|
self.assertTrue(
|
|
is_openeo_auth_configured(
|
|
OpenEOConnectionSettings(
|
|
auth_method="password",
|
|
username="user@example.com",
|
|
password="secret",
|
|
)
|
|
)
|
|
)
|
|
|
|
def test_verify_openeo_auth_command_skips_when_unconfigured(self):
|
|
stdout = StringIO()
|
|
|
|
call_command("verify_openeo_auth", "--skip-if-unconfigured", stdout=stdout)
|
|
|
|
self.assertIn("openEO auth check skipped", stdout.getvalue())
|
|
|
|
def test_connect_openeo_applies_proxy_to_session(self):
|
|
connection = Mock()
|
|
connection._get_oidc_provider.return_value = ("provider-1", None)
|
|
connection.get.return_value.json.return_value = {
|
|
"providers": [
|
|
{
|
|
"id": "provider-1",
|
|
"title": "Provider 1",
|
|
"issuer": "https://issuer.example.com",
|
|
"scopes": ["openid"],
|
|
"default_clients": [],
|
|
}
|
|
]
|
|
}
|
|
openeo_module = Mock()
|
|
openeo_module.connect.return_value = connection
|
|
oidc_module = Mock()
|
|
oidc_module.OidcClientCredentialsAuthenticator.return_value = Mock()
|
|
oidc_module.OidcClientInfo.side_effect = lambda **kwargs: kwargs
|
|
oidc_module.OidcProviderInfo.side_effect = lambda **kwargs: kwargs
|
|
oidc_module.OidcResourceOwnerPasswordAuthenticator.return_value = Mock()
|
|
connection._authenticate_oidc.return_value = connection
|
|
|
|
settings = OpenEOConnectionSettings(
|
|
backend_url="https://openeofed.dataspace.copernicus.eu",
|
|
auth_method="password",
|
|
timeout_seconds=123,
|
|
client_id="client-id",
|
|
username="user@example.com",
|
|
password="secret",
|
|
proxy_url="socks5h://127.0.0.1:10808",
|
|
)
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"openeo": openeo_module,
|
|
"openeo.rest": Mock(),
|
|
"openeo.rest.auth": Mock(),
|
|
"openeo.rest.auth.oidc": oidc_module,
|
|
},
|
|
):
|
|
connect_openeo(settings)
|
|
|
|
self.assertEqual(openeo_module.connect.call_args.kwargs["default_timeout"], 123)
|
|
session = openeo_module.connect.call_args.kwargs["session"]
|
|
self.assertEqual(session.proxies["https"], "socks5h://127.0.0.1:10808")
|
|
self.assertFalse(session.trust_env)
|
|
|
|
def test_timeout_override_session_logs_request_payload_before_dispatch(self):
|
|
response = Mock(status_code=200, headers={"Content-Type": "application/json"})
|
|
response.text = "{}"
|
|
response.url = "https://openeofed.dataspace.copernicus.eu/result"
|
|
with patch.object(requests.Session, "request", return_value=response) as request_mock:
|
|
with self.assertLogs("location_data.openeo_service", level="INFO") as captured:
|
|
session = build_openeo_requests_session(OpenEOConnectionSettings(proxy_url="socks5h://127.0.0.1:10808"))
|
|
session.request(
|
|
"post",
|
|
"https://openeofed.dataspace.copernicus.eu/result",
|
|
json={"process": {"foo": "bar"}},
|
|
headers={"Authorization": "Bearer secret"},
|
|
)
|
|
|
|
request_mock.assert_called_once()
|
|
self.assertTrue(any("openEO request payload" in line for line in captured.output))
|
|
self.assertTrue(any("***redacted***" in line for line in captured.output))
|
|
|
|
def test_connect_openeo_raises_clear_error_for_html_capabilities_response(self):
|
|
settings = OpenEOConnectionSettings(
|
|
backend_url="https://openeofed.dataspace.copernicus.eu",
|
|
timeout_seconds=600,
|
|
)
|
|
bad_response = Mock()
|
|
bad_response.url = "https://openeofed.dataspace.copernicus.eu/"
|
|
bad_response.headers = {"Content-Type": "text/html"}
|
|
bad_response.text = "<html>proxy page</html>"
|
|
|
|
session = build_openeo_requests_session(settings)
|
|
session.last_response_url = bad_response.url
|
|
session.last_response_content_type = "text/html"
|
|
session.last_response_preview = bad_response.text
|
|
|
|
openeo_module = Mock()
|
|
openeo_module.connect.side_effect = requests.exceptions.JSONDecodeError("Expecting value", "", 0)
|
|
|
|
oidc_module = Mock()
|
|
with patch("location_data.openeo_service.build_openeo_requests_session", return_value=session):
|
|
with patch.dict(
|
|
"sys.modules",
|
|
{
|
|
"openeo": openeo_module,
|
|
"openeo.rest": Mock(),
|
|
"openeo.rest.auth": Mock(),
|
|
"openeo.rest.auth.oidc": oidc_module,
|
|
},
|
|
):
|
|
with self.assertRaisesRegex(OpenEOServiceError, "non-JSON response"):
|
|
connect_openeo(settings)
|
|
|
|
def test_build_openeo_requests_session_mounts_retrying_adapters(self):
|
|
session = build_openeo_requests_session(
|
|
OpenEOConnectionSettings(
|
|
timeout_seconds=120,
|
|
http_retry_total=5,
|
|
http_retry_backoff_factor=2.0,
|
|
)
|
|
)
|
|
|
|
https_adapter = session.get_adapter("https://openeofed.dataspace.copernicus.eu")
|
|
self.assertEqual(https_adapter.max_retries.total, 5)
|
|
self.assertEqual(https_adapter.max_retries.connect, 5)
|
|
self.assertEqual(https_adapter.max_retries.backoff_factor, 2.0)
|
|
self.assertIsNone(https_adapter.max_retries.allowed_methods)
|
|
|
|
def test_run_aggregate_spatial_job_prefers_batch_job_results(self):
|
|
process = Mock()
|
|
job = Mock(job_id="job-123")
|
|
process.create_job.return_value = job
|
|
job.start_and_wait.return_value = job
|
|
results = Mock()
|
|
job.get_results.return_value = results
|
|
|
|
def write_json(target_dir):
|
|
Path(target_dir, "result.json").write_text('{"cell-1": {"mean": 0.5}}', encoding="utf-8")
|
|
|
|
results.download_files.side_effect = write_json
|
|
|
|
payload, job_ref = _run_aggregate_spatial_job(process, metric_name="ndvi")
|
|
|
|
self.assertEqual(payload, {"cell-1": {"mean": 0.5}})
|
|
self.assertEqual(job_ref, "job-123")
|
|
process.execute.assert_not_called()
|
|
|
|
def test_load_job_result_payload_archives_exact_raw_json_file(self):
|
|
job = Mock(job_id="job-123")
|
|
results = Mock()
|
|
job.get_results.return_value = results
|
|
raw_json = '{\n "cell-1": {"mean": 0.5}\n}\n'
|
|
|
|
def write_json(target_dir):
|
|
Path(target_dir, "timeseries.json").write_text(raw_json, encoding="utf-8")
|
|
|
|
results.download_files.side_effect = write_json
|
|
|
|
with TemporaryDirectory() as archive_dir:
|
|
with patch.dict(os.environ, {"OPENEO_PAYLOAD_ARCHIVE_DIR": archive_dir}, clear=False):
|
|
payload = _load_job_result_payload(job, metric_name="ndvi")
|
|
archive_path = Path(archive_dir) / "job-123__ndvi__timeseries.json"
|
|
self.assertTrue(archive_path.exists())
|
|
self.assertEqual(archive_path.read_text(encoding="utf-8"), raw_json)
|
|
|
|
self.assertEqual(payload, {"cell-1": {"mean": 0.5}})
|
|
|
|
def test_load_first_json_payload_prefers_stac_asset_data_over_metadata(self):
|
|
with TemporaryDirectory() as temp_dir:
|
|
Path(temp_dir, "item.json").write_text(
|
|
(
|
|
'{"stac_version":"1.0.0","assets":{"timeseries.json":{"href":"timeseries.json"}},'
|
|
'"extent":{"spatial":{},"temporal":{}}}'
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
Path(temp_dir, "timeseries.json").write_text('{"cell-1": {"mean": 0.5}}', encoding="utf-8")
|
|
|
|
payload = _load_first_json_payload(Path(temp_dir), job_ref="job-asset")
|
|
|
|
self.assertEqual(payload, {"cell-1": {"mean": 0.5}})
|
|
|
|
def test_load_first_json_payload_raises_clear_error_for_invalid_json(self):
|
|
with TemporaryDirectory() as temp_dir:
|
|
Path(temp_dir, "result.json").write_text("not-json", encoding="utf-8")
|
|
with self.assertRaises(OpenEOExecutionError):
|
|
with self.assertLogs("location_data.openeo_service", level="ERROR"):
|
|
_load_first_json_payload(Path(temp_dir), job_ref="job-123")
|