generate sitemap as part of deployment

This allows adding the lastmod field based on the last modification date
of the file.

3 files changed, 73 insertions, 109 deletions

diff --git a/deploy-static b/deploy-static
index 9d45560c..64086281 100755
--- a/deploy-static
+++ b/deploy-static
@@ -19,10 +19,16 @@ rsync -pcv --chmod=F644 --fsync --preallocate replicate.conf root@${servers[0]}:
 
 # use last modified timestamps from 0.grapheneos.org
 rsync -rptcv --chmod=D755,F644 --delete --fsync --preallocate root@${servers[0]}:/srv/grapheneos.org/ static-production
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-production/sitemap.xml{,.gz,.br} static-tmp/
 rsync -rpcv --chmod=D755,F644 --delete --fsync --preallocate static-tmp/ static-production
 for f in static-production/**.*(br|gz); do
     touch -r "${f%.*}" "$f"
 done
+./generate-sitemap
+xmllint --noblanks static-tmp/sitemap.xml --output static-tmp/sitemap.xml
+brotli -f static-tmp/sitemap.xml
+zopfli static-tmp/sitemap.xml
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-tmp/sitemap.xml{,.gz,.br} static-production/
 
 for server in ${servers[@]}; do
     echo $server
diff --git a/generate-sitemap b/generate-sitemap
new file mode 100755
index 00000000..91cc8408
--- /dev/null
+++ b/generate-sitemap
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+from datetime import datetime, timezone
+from os.path import getmtime
+
+base = "https://grapheneos.org"
+
+pages = [
+    ["/", 0.5],
+    ["/.well-known/security.txt", 0.0],
+    ["/LICENSE.txt", 0.0],
+    ["/articles/", 0.5],
+    ["/articles/attestation-compatibility-guide", 0.5],
+    ["/articles/grapheneos-servers", 0.1],
+    ["/articles/server-traffic-shaping", 0.5],
+    ["/articles/sitewide-advertising-industry-opt-out", 0.5],
+    ["/build", 0.5],
+    ["/camera-privacy-policy", 0.0],
+    ["/contact", 0.5],
+    ["/donate", 0.5],
+    ["/faq", 1.0],
+    ["/features", 1.0],
+    ["/history/", 0.3],
+    ["/history/copperheados", 0.1],
+    ["/history/legacy-changelog", 0.1],
+    ["/hiring", 0.2],
+    ["/humans.txt", 0.0],
+    ["/pdfviewer-privacy-policy", 0.0],
+    ["/install/", 0.5],
+    ["/install/cli", 0.5],
+    ["/install/web", 0.5],
+    ["/releases", 0.5],
+    ["/source", 0.5],
+    ["/usage", 1.0]
+]
+
+entries = []
+
+for page in pages:
+    path = page[0]
+    loc = base + path
+    filepath = "static-production" + path
+    if path[-1] == '/':
+        filepath += "index.html"
+    elif "." not in path:
+        filepath += ".html"
+
+    mtime = getmtime(filepath)
+    lastmod = datetime.fromtimestamp(mtime, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%:z")
+    priority = page[1]
+    entries.append(f"""
+    <url>
+        <loc>{loc}</loc>
+        <lastmod>{lastmod}</lastmod>
+        <priority>{priority}</priority>
+    </url>""")
+
+sitemap = f"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">{"".join(entries)}
+</urlset>
+"""
+
+with open("static-tmp/sitemap.xml", "w") as f:
+    f.write(sitemap)
diff --git a/static/sitemap.xml b/static/sitemap.xml
deleted file mode 100644
index 6fa73775..00000000
--- a/static/sitemap.xml
+++ /dev/null
@@ -1,109 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
-    <url>
-        <loc>https://grapheneos.org/</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/.well-known/security.txt</loc>
-        <priority>0.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/LICENSE.txt</loc>
-        <priority>0.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/articles/</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/articles/attestation-compatibility-guide</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/articles/grapheneos-servers</loc>
-        <priority>0.1</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/articles/server-traffic-shaping</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/articles/sitewide-advertising-industry-opt-out</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/build</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/camera-privacy-policy</loc>
-        <priority>0.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/contact</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/donate</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/faq</loc>
-        <priority>1.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/features</loc>
-        <priority>1.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/history/</loc>
-        <priority>0.3</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/history/copperheados</loc>
-        <priority>0.1</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/history/legacy-changelog</loc>
-        <priority>0.1</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/hiring</loc>
-        <priority>0.2</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/humans.txt</loc>
-        <priority>0.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/pdfviewer-privacy-policy</loc>
-        <priority>0.0</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/install/</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/install/cli</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/install/web</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/releases</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/source</loc>
-        <priority>0.5</priority>
-    </url>
-    <url>
-        <loc>https://grapheneos.org/usage</loc>
-        <priority>1.0</priority>
-    </url>
-</urlset>