Initial commit on Forgejo

Fresh repository history for elektrine/tarakan hosted at
https://git.elektrine.com/elektrine/tarakan.

lib/tarakan_web/controllers/seo_controller.ex

@@ -0,0 +1,289 @@
+defmodule TarakanWeb.SEOController do
+  @moduledoc """
+  Search-engine discovery endpoints.
+
+  Streams listed repos and findings via cursors. Single-file sitemap stays
+  under 50k URLs; multi-file index kicks in at 20k listed repositories.
+  """
+
+  use TarakanWeb, :controller
+
+  alias Tarakan.Repositories
+  alias Tarakan.Scans
+  alias Tarakan.Work
+  alias TarakanWeb.RepositoryPaths
+
+  @max_urls 50_000
+  @multi_file_listed_threshold 20_000
+  @child_page_size 10_000
+
+  def robots(conn, _params) do
+    body = """
+    User-agent: *
+    Allow: /
+
+    Disallow: /*/code/
+    Disallow: /findings/*/code
+
+    Sitemap: #{url(~p"/sitemap.xml")}
+    """
+
+    conn
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, body)
+  end
+
+  def security_txt(conn, _params) do
+    contact = Application.get_env(:tarakan, :security_contact, "security@example.com")
+    base = TarakanWeb.Endpoint.url()
+
+    expires =
+      DateTime.utc_now()
+      |> DateTime.add(365, :day)
+      |> DateTime.truncate(:second)
+      |> DateTime.to_iso8601()
+
+    body = """
+    Contact: mailto:#{contact}
+    Expires: #{expires}
+    Preferred-Languages: en
+    Canonical: #{base}/.well-known/security.txt
+    Policy: #{base}/policies/disclosure
+    """
+
+    conn
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, body)
+  end
+
+  def sitemap(conn, _params) do
+    listed = Repositories.registry_stats().repositories || 0
+
+    if listed >= @multi_file_listed_threshold do
+      sitemap_index(conn)
+    else
+      sitemap_single(conn)
+    end
+  end
+
+  def sitemap_hubs(conn, _params) do
+    base = TarakanWeb.Endpoint.url()
+    now = DateTime.utc_now()
+
+    entries =
+      hub_entries(now)
+      |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end)
+
+    send_urlset(conn, entries)
+  end
+
+  def sitemap_repos(conn, %{"page" => page}) do
+    page = parse_page(page)
+    base = TarakanWeb.Endpoint.url()
+    offset = (page - 1) * @child_page_size
+
+    {entries, _} =
+      Repositories.stream_listed_repositories(limit: 1_000)
+      |> Enum.reduce_while({[], 0}, fn repo, {acc, i} ->
+        cond do
+          i < offset ->
+            {:cont, {acc, i + 1}}
+
+          length(acc) >= @child_page_size ->
+            {:halt, {acc, i}}
+
+          true ->
+            entry =
+              {RepositoryPaths.repository_security_path(repo), repo.updated_at, "0.8"}
+
+            {:cont, {[entry | acc], i + 1}}
+        end
+      end)
+
+    xml_entries =
+      entries
+      |> Enum.reverse()
+      |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end)
+
+    send_urlset(conn, xml_entries)
+  end
+
+  def sitemap_findings(conn, %{"page" => page}) do
+    page = parse_page(page)
+    base = TarakanWeb.Endpoint.url()
+    offset = (page - 1) * @child_page_size
+
+    {entries, _} =
+      Scans.stream_indexable_findings()
+      |> Enum.reduce_while({[], 0}, fn finding, {acc, i} ->
+        cond do
+          i < offset ->
+            {:cont, {acc, i + 1}}
+
+          length(acc) >= @child_page_size ->
+            {:halt, {acc, i}}
+
+          true ->
+            priority =
+              cond do
+                Map.get(finding, :verified) == true -> "0.85"
+                (Map.get(finding, :confirmations_count) || 0) > 0 -> "0.75"
+                true -> "0.65"
+              end
+
+            entry = {~p"/findings/#{finding.public_id}", finding.updated_at, priority}
+            {:cont, {[entry | acc], i + 1}}
+        end
+      end)
+
+    xml_entries =
+      entries
+      |> Enum.reverse()
+      |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end)
+
+    send_urlset(conn, xml_entries)
+  end
+
+  def sitemap_jobs(conn, %{"page" => page}) do
+    page = parse_page(page)
+    base = TarakanWeb.Endpoint.url()
+
+    entries =
+      Work.list_indexable_public_tasks()
+      |> Enum.drop((page - 1) * @child_page_size)
+      |> Enum.take(@child_page_size)
+      |> Enum.map(fn task ->
+        priority = if task.kind == "verify_findings", do: "0.8", else: "0.7"
+        url_entry(base, ~p"/jobs/#{task.id}", task.updated_at, priority)
+      end)
+
+    send_urlset(conn, entries)
+  end
+
+  defp sitemap_index(conn) do
+    base = TarakanWeb.Endpoint.url()
+    listed = Repositories.registry_stats().repositories || 0
+    repo_pages = max(ceil(listed / @child_page_size), 1)
+    # Findings unknown without full count; emit a few pages (crawlers stop on empty).
+    finding_pages = 5
+    job_pages = 1
+
+    locs =
+      ["#{base}/sitemap/hubs.xml"] ++
+        Enum.map(1..repo_pages, &"#{base}/sitemap/repos/#{&1}") ++
+        Enum.map(1..finding_pages, &"#{base}/sitemap/findings/#{&1}") ++
+        Enum.map(1..job_pages, &"#{base}/sitemap/jobs/#{&1}")
+
+    body = [
+      ~s(<?xml version="1.0" encoding="UTF-8"?>),
+      ~s(<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">),
+      Enum.map(locs, fn loc ->
+        ["<sitemap><loc>", xml_escape(loc), "</loc></sitemap>"]
+      end),
+      "</sitemapindex>"
+    ]
+
+    conn
+    |> put_resp_content_type("text/xml")
+    |> send_resp(200, body)
+  end
+
+  defp sitemap_single(conn) do
+    base = TarakanWeb.Endpoint.url()
+    now = DateTime.utc_now()
+
+    hub_entries = hub_entries(now)
+
+    repository_entries =
+      Repositories.stream_listed_repositories()
+      |> Enum.map(fn repository ->
+        {RepositoryPaths.repository_security_path(repository), repository.updated_at, "0.8"}
+      end)
+
+    finding_entries =
+      Scans.stream_indexable_findings()
+      |> Enum.map(fn finding ->
+        priority =
+          cond do
+            Map.get(finding, :verified) == true -> "0.85"
+            (Map.get(finding, :confirmations_count) || 0) > 0 -> "0.75"
+            true -> "0.65"
+          end
+
+        {~p"/findings/#{finding.public_id}", finding.updated_at, priority}
+      end)
+
+    job_entries =
+      Enum.map(Work.list_indexable_public_tasks(), fn task ->
+        priority = if task.kind == "verify_findings", do: "0.8", else: "0.7"
+        {~p"/jobs/#{task.id}", task.updated_at, priority}
+      end)
+
+    entries =
+      (hub_entries ++ finding_entries ++ job_entries ++ repository_entries)
+      |> Enum.take(@max_urls)
+      |> Enum.map(fn
+        {path, lastmod, priority} -> url_entry(base, path, lastmod, priority)
+        {path, lastmod} -> url_entry(base, path, lastmod, "0.5")
+      end)
+
+    send_urlset(conn, entries)
+  end
+
+  defp hub_entries(now) do
+    [
+      {"/explore", now, "1.0"},
+      {"/", now, "0.9"},
+      {"/jobs", now, "0.9"},
+      {"/agents", now, "0.9"},
+      {"/infestations", now, "0.85"},
+      {"/leaderboard", now, "0.7"}
+    ]
+  end
+
+  defp send_urlset(conn, entries) do
+    xml = [
+      ~s(<?xml version="1.0" encoding="UTF-8"?>),
+      ~s(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">),
+      entries,
+      "</urlset>"
+    ]
+
+    conn
+    |> put_resp_content_type("text/xml")
+    |> send_resp(200, xml)
+  end
+
+  defp parse_page(page) do
+    case Integer.parse(to_string(page)) do
+      {n, _} when n >= 1 -> n
+      _ -> 1
+    end
+  end
+
+  defp url_entry(base, path, lastmod, priority) do
+    [
+      "<url><loc>",
+      xml_escape(base <> path),
+      "</loc>",
+      lastmod_element(lastmod),
+      "<priority>",
+      priority,
+      "</priority>",
+      "</url>"
+    ]
+  end
+
+  defp lastmod_element(nil), do: []
+
+  defp lastmod_element(%DateTime{} = datetime) do
+    ["<lastmod>", datetime |> DateTime.to_date() |> Date.to_iso8601(), "</lastmod>"]
+  end
+
+  defp xml_escape(text) do
+    text
+    |> String.replace("&", "&amp;")
+    |> String.replace("<", "&lt;")
+    |> String.replace(">", "&gt;")
+  end
+end