defmodule Tarakan.ModelAnalyticsTest do
  use Tarakan.DataCase, async: true

  alias Tarakan.ModelAnalytics
  alias Tarakan.Repo
  alias Tarakan.Scans.CanonicalFinding

  defp scan_with_model(repository, account, model, findings_json) do
    scan_fixture(repository, account, %{
      "model" => model,
      "findings_json" => findings_json
    })
  end

  defp set_status(scan, status) do
    for occurrence <- scan.findings do
      CanonicalFinding
      |> Repo.get!(occurrence.canonical_finding_id)
      |> Ecto.Changeset.change(status: status)
      |> Repo.update!()
    end
  end

  describe "model_scoreboard/1" do
    test "counts findings per model and scores precision on settled ones" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      confirmed = scan_with_model(repository, account, "claude-opus-5", findings_json_fixture(2))
      set_status(confirmed, "verified")

      # A second model reports a different set that the record rejects.
      disputed =
        scan_with_model(
          repository,
          account,
          "noisy-model-1",
          Jason.encode!(%{
            "tarakan_scan_format" => 1,
            "findings" => [
              %{
                "file" => "lib/other.ex",
                "line_start" => 3,
                "line_end" => 4,
                "severity" => "low",
                "title" => "Speculative issue",
                "description" => "Probably nothing."
              }
            ]
          })
        )

      set_status(disputed, "disputed")

      rows = ModelAnalytics.model_scoreboard(min_findings: 1)

      opus = Enum.find(rows, &(&1.model == "claude-opus-5"))
      noisy = Enum.find(rows, &(&1.model == "noisy-model-1"))

      assert opus.findings == 2
      assert opus.confirmed == 2
      assert opus.disputed == 0
      assert opus.precision == 100.0

      assert noisy.findings == 1
      assert noisy.confirmed == 0
      assert noisy.precision == 0.0
    end

    test "hides models below the reporting floor" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      scan_with_model(repository, account, "rare-model", findings_json_fixture(1))

      assert ModelAnalytics.model_scoreboard(min_findings: 5) == []
      assert [_row] = ModelAnalytics.model_scoreboard(min_findings: 1)
    end
  end

  describe "blind_spots/2" do
    test "lists confirmed findings a model had the chance to report and did not" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      found = scan_with_model(repository, account, "sharp-model", findings_json_fixture(1))
      set_status(found, "verified")

      # The other model scanned the same repository but reported something else.
      scan_with_model(
        repository,
        account,
        "blunt-model",
        Jason.encode!(%{
          "tarakan_scan_format" => 1,
          "findings" => [
            %{
              "file" => "lib/unrelated.ex",
              "line_start" => 99,
              "line_end" => 100,
              "severity" => "low",
              "title" => "Something else entirely",
              "description" => "Unrelated."
            }
          ]
        })
      )

      assert [missed] = ModelAnalytics.blind_spots("blunt-model")
      assert missed.title == hd(found.findings).title

      # The model that found it is not blind to it.
      assert ModelAnalytics.blind_spots("sharp-model") == []
    end

    test "a model is not blamed for repositories it never scanned" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      found = scan_with_model(repository, account, "present-model", findings_json_fixture(1))
      set_status(found, "verified")

      assert ModelAnalytics.blind_spots("absent-model") == []
    end
  end

  describe "disagreement_patterns/1" do
    test "surfaces classes that more than one model reported" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      # Both models review the same code and land on the same class, so the
      # occurrences share one canonical finding.
      first = scan_with_model(repository, account, "model-a", findings_json_fixture(1))
      _second = scan_with_model(repository, account, "model-b", findings_json_fixture(1))

      set_status(first, "disputed")

      assert [row] = ModelAnalytics.disagreement_patterns()
      assert row.models == 2
      assert row.findings == 1
      assert row.confirmed == 0
      assert row.disputed == 1
    end

    test "one model agreeing with itself is not a disagreement" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      scan = scan_with_model(repository, account, "solo-model", findings_json_fixture(1))
      set_status(scan, "verified")

      assert ModelAnalytics.disagreement_patterns() == []
    end
  end

  describe "account_model_scoreboard/2" do
    test "scores a contributor's own agents against their own submissions" do
      account = github_account_fixture()
      other = github_account_fixture()
      repository = github_repository_fixture(account)

      good = scan_with_model(repository, account, "my-good-agent", findings_json_fixture(2))
      set_status(good, "verified")

      bad =
        scan_with_model(
          repository,
          account,
          "my-noisy-agent",
          Jason.encode!(%{
            "tarakan_scan_format" => 1,
            "findings" => [
              %{
                "file" => "lib/other.ex",
                "line_start" => 3,
                "line_end" => 4,
                "severity" => "low",
                "title" => "Speculative",
                "description" => "Probably nothing."
              }
            ]
          })
        )

      set_status(bad, "disputed")

      # Somebody else's run of the same model must not show up here.
      stranger = scan_with_model(repository, other, "my-good-agent", findings_json_fixture(3))
      set_status(stranger, "verified")

      rows = ModelAnalytics.account_model_scoreboard(account.id)

      good_row = Enum.find(rows, &(&1.model == "my-good-agent"))
      noisy_row = Enum.find(rows, &(&1.model == "my-noisy-agent"))

      assert good_row.findings == 2
      assert good_row.precision == 100.0
      assert noisy_row.precision == 0.0
    end

    test "applies no minimum, so a first finding still counts" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      scan_with_model(repository, account, "fresh-agent", findings_json_fixture(1))

      assert [row] = ModelAnalytics.account_model_scoreboard(account.id)
      assert row.findings == 1
    end

    test "is empty for an account that has submitted nothing" do
      assert ModelAnalytics.account_model_scoreboard(github_account_fixture().id) == []
    end
  end

  describe "known_models/0" do
    test "lists models that reported publicly" do
      account = github_account_fixture()
      repository = github_repository_fixture(account)

      scan_with_model(repository, account, "listed-model", findings_json_fixture(1))

      assert "listed-model" in ModelAnalytics.known_models()
    end
  end
end
