diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 77da7db87e409..74e4279a6ab83 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -61,6 +61,7 @@ datafusion/proto-common/src/generated/prost.rs .github/ISSUE_TEMPLATE/feature_request.yml .github/workflows/docs.yaml docs/source/llms.txt +docs/source/robots.txt **/node_modules/* datafusion/wasmtest/pkg/* clippy.toml diff --git a/docs/source/conf.py b/docs/source/conf.py index 81ffce3ad1c2b..68ba628bba1f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ # Copy agent-facing files (llms.txt) verbatim to the site root so they # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. -html_extra_path = ["llms.txt"] +html_extra_path = ["llms.txt", "robots.txt"] # Base URL of the published site. Required by sphinx-sitemap to build # absolute entries in the generated sitemap.xml. diff --git a/docs/source/robots.txt b/docs/source/robots.txt new file mode 100644 index 0000000000000..9c80d8219c4c9 --- /dev/null +++ b/docs/source/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://datafusion.apache.org/sitemap.xml