Ai/location_data/test_openeo_service.py

from decimal import Decimal
from io import StringIO
import os
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch

from django.core.management import call_command
from django.test import SimpleTestCase
import requests

from config.proxy import resolve_requests_proxy_url
from location_data.openeo_service import (
    OpenEOConnectionSettings,
    OpenEOServiceError,
    OpenEOExecutionError,
    _log_raw_payload_summary,
    _load_first_json_payload,
    _resolve_openeo_proxy_url_from_env,
    _run_aggregate_spatial_job,
    log_openeo_request_summary,
    build_openeo_requests_session,
    build_empty_metric_payload,
    connect_openeo,
    is_openeo_auth_configured,
    linear_to_db,
    merge_metric_results,
    parse_aggregate_spatial_response,
)


class OpenEOServiceParsingTests(SimpleTestCase):
    def test_parse_feature_collection_results(self):
        payload = {
            "type": "FeatureCollection",
            "features": [
                {
                    "type": "Feature",
                    "id": "cell-1",
                    "properties": {"mean": 0.61},
                },
                {
                    "type": "Feature",
                    "id": "cell-2",
                    "properties": {"mean": 0.47},
                },
            ],
        }

        result = parse_aggregate_spatial_response(payload, "ndvi")

        self.assertEqual(result["cell-1"]["ndvi"], 0.61)
        self.assertEqual(result["cell-2"]["ndvi"], 0.47)

    def test_parse_mapping_results(self):
        payload = {
            "cell-1": {"mean": 12.4},
            "cell-2": {"mean": 15.1},
        }

        result = parse_aggregate_spatial_response(payload, "lst_c")

        self.assertEqual(result["cell-1"]["lst_c"], 12.4)
        self.assertEqual(result["cell-2"]["lst_c"], 15.1)

    def test_parse_mapping_results_maps_numeric_keys_to_expected_feature_ids(self):
        payload = {
            "0": {"mean": 12.4},
            "1": {"mean": 15.1},
        }

        result = parse_aggregate_spatial_response(
            payload,
            "lst_c",
            expected_feature_ids=["cell-1", "cell-2"],
        )

        self.assertEqual(result["cell-1"]["lst_c"], 12.4)
        self.assertEqual(result["cell-2"]["lst_c"], 15.1)

    def test_parse_list_results_maps_positional_payload_to_expected_feature_ids(self):
        payload = [{"mean": 0.61}, {"mean": 0.47}]

        result = parse_aggregate_spatial_response(
            payload,
            "ndvi",
            expected_feature_ids=["cell-1", "cell-2"],
        )

        self.assertEqual(result["cell-1"]["ndvi"], 0.61)
        self.assertEqual(result["cell-2"]["ndvi"], 0.47)

    def test_log_raw_payload_summary_warns_for_empty_payload(self):
        with self.assertLogs("location_data.openeo_service", level="WARNING") as captured:
            summary = _log_raw_payload_summary({}, metric_name="ndvi", job_ref="job-1")

        self.assertEqual(summary["returned_cell_count"], 0)
        self.assertIn("openEO payload is empty for job_ref=job-1", "\n".join(captured.output))

    def test_parse_logs_feature_mismatch_for_unexpected_keys(self):
        payload = {"cell-1": {"foo": 12.4}}

        with self.assertLogs("location_data.openeo_service", level="WARNING") as captured:
            result = parse_aggregate_spatial_response(payload, "lst_c", job_ref="job-2")

        self.assertEqual(result["cell-1"]["lst_c"], 12.4)
        self.assertIn("Feature mismatch for cell=cell-1, available_keys=['foo']", "\n".join(captured.output))

    def test_linear_to_db(self):
        self.assertEqual(linear_to_db(10.0), 10.0)
        self.assertEqual(linear_to_db(Decimal("1.0")), 0.0)
        self.assertIsNone(linear_to_db(0))
        self.assertIsNone(linear_to_db(-1))

    def test_merge_metric_results(self):
        target = {"cell-1": build_empty_metric_payload()}

        merge_metric_results(
            target,
            {
                "cell-1": {"ndvi": 0.5},
                "cell-2": {"ndwi": 0.2},
            },
        )

        self.assertEqual(target["cell-1"]["ndvi"], 0.5)
        self.assertEqual(target["cell-2"]["ndwi"], 0.2)
        self.assertIn("soil_vv_db", target["cell-2"])

    def test_log_openeo_request_summary_logs_expected_fields(self):
        cell = Mock()
        cell.cell_code = "cell-1"
        with self.assertLogs("location_data.openeo_service", level="INFO") as captured:
            log_openeo_request_summary(
                cells=[cell],
                temporal_start="2026-04-08",
                temporal_end="2026-05-08",
                spatial_extent={"west": 49.9995, "south": 49.9995, "east": 50.0005, "north": 50.0005},
                selected_features=["ndvi", "ndwi"],
            )

        joined = "\n".join(captured.output)
        self.assertIn("openEO request summary", joined)
        self.assertIn('"cell_count": 1', joined)
        self.assertIn('"date_range_days": 31', joined)
        self.assertIn('"metrics": ["ndvi", "ndwi"]', joined)


class OpenEOConnectionTests(SimpleTestCase):
    def test_default_openeo_timeout_is_ten_minutes(self):
        with patch.dict(os.environ, {}, clear=True):
            settings = OpenEOConnectionSettings.from_env()

        self.assertEqual(settings.timeout_seconds, 600.0)

    def test_default_openeo_proxy_url_uses_proxychains_endpoint_without_wrapping_process(self):
        with patch.dict(
            os.environ,
            {
                "ENABLE_PROXYCHAINS": "0",
                "PROXYCHAINS_PROXY_TYPE": "socks4",
                "PROXYCHAINS_PROXY_HOST": "host.docker.internal",
                "PROXYCHAINS_PROXY_PORT": "10808",
                "OPENEO_PROXY_URL": "socks5h://host.docker.internal:10808",
            },
            clear=False,
        ):
            self.assertEqual(
                _resolve_openeo_proxy_url_from_env(),
                "socks4a://host.docker.internal:10808",
            )

    def test_requests_proxy_is_disabled_when_proxychains_targets_same_endpoint(self):
        with patch.dict(
            os.environ,
            {
                "ENABLE_PROXYCHAINS": "1",
                "PROXYCHAINS_PROXY_TYPE": "socks4",
                "PROXYCHAINS_PROXY_HOST": "host.docker.internal",
                "PROXYCHAINS_PROXY_PORT": "10808",
            },
            clear=False,
        ):
            self.assertEqual(
                resolve_requests_proxy_url("socks5h://host.docker.internal:10808"),
                "",
            )

    def test_is_openeo_auth_configured_for_client_credentials(self):
        self.assertTrue(
            is_openeo_auth_configured(
                OpenEOConnectionSettings(
                    auth_method="client_credentials",
                    client_id="client-id",
                    client_secret="client-secret",
                )
            )
        )

    def test_is_openeo_auth_configured_for_password(self):
        self.assertTrue(
            is_openeo_auth_configured(
                OpenEOConnectionSettings(
                    auth_method="password",
                    username="user@example.com",
                    password="secret",
                )
            )
        )

    def test_verify_openeo_auth_command_skips_when_unconfigured(self):
        stdout = StringIO()

        call_command("verify_openeo_auth", "--skip-if-unconfigured", stdout=stdout)

        self.assertIn("openEO auth check skipped", stdout.getvalue())

    def test_connect_openeo_applies_proxy_to_session(self):
        connection = Mock()
        connection._get_oidc_provider.return_value = ("provider-1", None)
        connection.get.return_value.json.return_value = {
            "providers": [
                {
                    "id": "provider-1",
                    "title": "Provider 1",
                    "issuer": "https://issuer.example.com",
                    "scopes": ["openid"],
                    "default_clients": [],
                }
            ]
        }
        openeo_module = Mock()
        openeo_module.connect.return_value = connection
        oidc_module = Mock()
        oidc_module.OidcClientCredentialsAuthenticator.return_value = Mock()
        oidc_module.OidcClientInfo.side_effect = lambda **kwargs: kwargs
        oidc_module.OidcProviderInfo.side_effect = lambda **kwargs: kwargs
        oidc_module.OidcResourceOwnerPasswordAuthenticator.return_value = Mock()
        connection._authenticate_oidc.return_value = connection

        settings = OpenEOConnectionSettings(
            backend_url="https://openeofed.dataspace.copernicus.eu",
            auth_method="password",
            timeout_seconds=123,
            client_id="client-id",
            username="user@example.com",
            password="secret",
            proxy_url="socks5h://127.0.0.1:10808",
        )
        with patch.dict(
            "sys.modules",
            {
                "openeo": openeo_module,
                "openeo.rest": Mock(),
                "openeo.rest.auth": Mock(),
                "openeo.rest.auth.oidc": oidc_module,
            },
        ):
            connect_openeo(settings)

        self.assertEqual(openeo_module.connect.call_args.kwargs["default_timeout"], 123)
        session = openeo_module.connect.call_args.kwargs["session"]
        self.assertEqual(session.proxies["https"], "socks5h://127.0.0.1:10808")
        self.assertFalse(session.trust_env)

    def test_timeout_override_session_logs_request_payload_before_dispatch(self):
        response = Mock(status_code=200, headers={"Content-Type": "application/json"})
        response.text = "{}"
        response.url = "https://openeofed.dataspace.copernicus.eu/result"
        with patch.object(requests.Session, "request", return_value=response) as request_mock:
            with self.assertLogs("location_data.openeo_service", level="INFO") as captured:
                session = build_openeo_requests_session(OpenEOConnectionSettings(proxy_url="socks5h://127.0.0.1:10808"))
                session.request(
                    "post",
                    "https://openeofed.dataspace.copernicus.eu/result",
                    json={"process": {"foo": "bar"}},
                    headers={"Authorization": "Bearer secret"},
                )

        request_mock.assert_called_once()
        self.assertTrue(any("openEO request payload" in line for line in captured.output))
        self.assertTrue(any("***redacted***" in line for line in captured.output))

    def test_connect_openeo_raises_clear_error_for_html_capabilities_response(self):
        settings = OpenEOConnectionSettings(
            backend_url="https://openeofed.dataspace.copernicus.eu",
            timeout_seconds=600,
        )
        bad_response = Mock()
        bad_response.url = "https://openeofed.dataspace.copernicus.eu/"
        bad_response.headers = {"Content-Type": "text/html"}
        bad_response.text = "<html>proxy page</html>"

        session = build_openeo_requests_session(settings)
        session.last_response_url = bad_response.url
        session.last_response_content_type = "text/html"
        session.last_response_preview = bad_response.text

        openeo_module = Mock()
        openeo_module.connect.side_effect = requests.exceptions.JSONDecodeError("Expecting value", "", 0)

        oidc_module = Mock()
        with patch("location_data.openeo_service.build_openeo_requests_session", return_value=session):
            with patch.dict(
                "sys.modules",
                {
                    "openeo": openeo_module,
                    "openeo.rest": Mock(),
                    "openeo.rest.auth": Mock(),
                    "openeo.rest.auth.oidc": oidc_module,
                },
            ):
                with self.assertRaisesRegex(OpenEOServiceError, "non-JSON response"):
                    connect_openeo(settings)

    def test_build_openeo_requests_session_mounts_retrying_adapters(self):
        session = build_openeo_requests_session(
            OpenEOConnectionSettings(
                timeout_seconds=120,
                http_retry_total=5,
                http_retry_backoff_factor=2.0,
            )
        )

        https_adapter = session.get_adapter("https://openeofed.dataspace.copernicus.eu")
        self.assertEqual(https_adapter.max_retries.total, 5)
        self.assertEqual(https_adapter.max_retries.connect, 5)
        self.assertEqual(https_adapter.max_retries.backoff_factor, 2.0)
        self.assertIsNone(https_adapter.max_retries.allowed_methods)

    def test_run_aggregate_spatial_job_prefers_batch_job_results(self):
        process = Mock()
        job = Mock(job_id="job-123")
        process.create_job.return_value = job
        job.start_and_wait.return_value = job
        results = Mock()
        job.get_results.return_value = results

        def write_json(target_dir):
            Path(target_dir, "result.json").write_text('{"cell-1": {"mean": 0.5}}', encoding="utf-8")

        results.download_files.side_effect = write_json

        payload, job_ref = _run_aggregate_spatial_job(process, metric_name="ndvi")

        self.assertEqual(payload, {"cell-1": {"mean": 0.5}})
        self.assertEqual(job_ref, "job-123")
        process.execute.assert_not_called()

    def test_load_first_json_payload_prefers_stac_asset_data_over_metadata(self):
        with TemporaryDirectory() as temp_dir:
            Path(temp_dir, "item.json").write_text(
                (
                    '{"stac_version":"1.0.0","assets":{"timeseries.json":{"href":"timeseries.json"}},'
                    '"extent":{"spatial":{},"temporal":{}}}'
                ),
                encoding="utf-8",
            )
            Path(temp_dir, "timeseries.json").write_text('{"cell-1": {"mean": 0.5}}', encoding="utf-8")

            payload = _load_first_json_payload(Path(temp_dir), job_ref="job-asset")

        self.assertEqual(payload, {"cell-1": {"mean": 0.5}})

    def test_load_first_json_payload_raises_clear_error_for_invalid_json(self):
        with TemporaryDirectory() as temp_dir:
            Path(temp_dir, "result.json").write_text("not-json", encoding="utf-8")
            with self.assertRaises(OpenEOExecutionError):
                with self.assertLogs("location_data.openeo_service", level="ERROR"):
                    _load_first_json_payload(Path(temp_dir), job_ref="job-123")