defmodule TarakanWeb.SEOController do @moduledoc """ Search-engine discovery endpoints. Streams listed repos and findings via cursors. Single-file sitemap stays under 50k URLs; multi-file index kicks in at 20k listed repositories. """ use TarakanWeb, :controller alias Tarakan.Repositories alias Tarakan.Scans alias Tarakan.Work alias TarakanWeb.RepositoryPaths @max_urls 50_000 @multi_file_listed_threshold 20_000 @child_page_size 10_000 def robots(conn, _params) do body = """ User-agent: * Allow: / Disallow: /*/code/ Disallow: /findings/*/code Sitemap: #{url(~p"/sitemap.xml")} """ conn |> put_resp_content_type("text/plain") |> send_resp(200, body) end def security_txt(conn, _params) do contact = Application.get_env(:tarakan, :security_contact, "security@example.com") base = TarakanWeb.Endpoint.url() expires = DateTime.utc_now() |> DateTime.add(365, :day) |> DateTime.truncate(:second) |> DateTime.to_iso8601() body = """ Contact: mailto:#{contact} Expires: #{expires} Preferred-Languages: en Canonical: #{base}/.well-known/security.txt Policy: #{base}/policies/disclosure """ conn |> put_resp_content_type("text/plain") |> send_resp(200, body) end def sitemap(conn, _params) do listed = Repositories.registry_stats().repositories || 0 if listed >= @multi_file_listed_threshold do sitemap_index(conn) else sitemap_single(conn) end end def sitemap_hubs(conn, _params) do base = TarakanWeb.Endpoint.url() now = DateTime.utc_now() entries = hub_entries(now) |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end) send_urlset(conn, entries) end def sitemap_repos(conn, %{"page" => page}) do page = parse_page(page) base = TarakanWeb.Endpoint.url() offset = (page - 1) * @child_page_size {entries, _} = Repositories.stream_listed_repositories(limit: 1_000) |> Enum.reduce_while({[], 0}, fn repo, {acc, i} -> cond do i < offset -> {:cont, {acc, i + 1}} length(acc) >= @child_page_size -> {:halt, {acc, i}} true -> entry = {RepositoryPaths.repository_security_path(repo), repo.updated_at, "0.8"} {:cont, {[entry | acc], i + 1}} end end) xml_entries = entries |> Enum.reverse() |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end) send_urlset(conn, xml_entries) end def sitemap_findings(conn, %{"page" => page}) do page = parse_page(page) base = TarakanWeb.Endpoint.url() offset = (page - 1) * @child_page_size {entries, _} = Scans.stream_indexable_findings() |> Enum.reduce_while({[], 0}, fn finding, {acc, i} -> cond do i < offset -> {:cont, {acc, i + 1}} length(acc) >= @child_page_size -> {:halt, {acc, i}} true -> priority = cond do Map.get(finding, :verified) == true -> "0.85" (Map.get(finding, :confirmations_count) || 0) > 0 -> "0.75" true -> "0.65" end entry = {~p"/findings/#{finding.public_id}", finding.updated_at, priority} {:cont, {[entry | acc], i + 1}} end end) xml_entries = entries |> Enum.reverse() |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) end) send_urlset(conn, xml_entries) end def sitemap_jobs(conn, %{"page" => page}) do page = parse_page(page) base = TarakanWeb.Endpoint.url() entries = Work.list_indexable_public_tasks() |> Enum.drop((page - 1) * @child_page_size) |> Enum.take(@child_page_size) |> Enum.map(fn task -> priority = if task.kind == "verify_findings", do: "0.8", else: "0.7" url_entry(base, ~p"/jobs/#{task.id}", task.updated_at, priority) end) send_urlset(conn, entries) end defp sitemap_index(conn) do base = TarakanWeb.Endpoint.url() listed = Repositories.registry_stats().repositories || 0 repo_pages = max(ceil(listed / @child_page_size), 1) # Findings unknown without full count; emit a few pages (crawlers stop on empty). finding_pages = 5 job_pages = 1 locs = ["#{base}/sitemap/hubs.xml"] ++ Enum.map(1..repo_pages, &"#{base}/sitemap/repos/#{&1}") ++ Enum.map(1..finding_pages, &"#{base}/sitemap/findings/#{&1}") ++ Enum.map(1..job_pages, &"#{base}/sitemap/jobs/#{&1}") body = [ ~s(), ~s(), Enum.map(locs, fn loc -> ["", xml_escape(loc), ""] end), "" ] conn |> put_resp_content_type("text/xml") |> send_resp(200, body) end defp sitemap_single(conn) do base = TarakanWeb.Endpoint.url() now = DateTime.utc_now() hub_entries = hub_entries(now) repository_entries = Repositories.stream_listed_repositories() |> Enum.map(fn repository -> {RepositoryPaths.repository_security_path(repository), repository.updated_at, "0.8"} end) finding_entries = Scans.stream_indexable_findings() |> Enum.map(fn finding -> priority = cond do Map.get(finding, :verified) == true -> "0.85" (Map.get(finding, :confirmations_count) || 0) > 0 -> "0.75" true -> "0.65" end {~p"/findings/#{finding.public_id}", finding.updated_at, priority} end) job_entries = Enum.map(Work.list_indexable_public_tasks(), fn task -> priority = if task.kind == "verify_findings", do: "0.8", else: "0.7" {~p"/jobs/#{task.id}", task.updated_at, priority} end) entries = (hub_entries ++ finding_entries ++ job_entries ++ repository_entries) |> Enum.take(@max_urls) |> Enum.map(fn {path, lastmod, priority} -> url_entry(base, path, lastmod, priority) {path, lastmod} -> url_entry(base, path, lastmod, "0.5") end) send_urlset(conn, entries) end defp hub_entries(now) do [ {"/explore", now, "1.0"}, {"/", now, "0.9"}, {"/jobs", now, "0.9"}, {"/agents", now, "0.9"}, {"/infestations", now, "0.85"}, {"/leaderboard", now, "0.7"} ] end defp send_urlset(conn, entries) do xml = [ ~s(), ~s(), entries, "" ] conn |> put_resp_content_type("text/xml") |> send_resp(200, xml) end defp parse_page(page) do case Integer.parse(to_string(page)) do {n, _} when n >= 1 -> n _ -> 1 end end defp url_entry(base, path, lastmod, priority) do [ "", xml_escape(base <> path), "", lastmod_element(lastmod), "", priority, "", "" ] end defp lastmod_element(nil), do: [] defp lastmod_element(%DateTime{} = datetime) do ["", datetime |> DateTime.to_date() |> Date.to_iso8601(), ""] end defp xml_escape(text) do text |> String.replace("&", "&") |> String.replace("<", "<") |> String.replace(">", ">") end end