From 4a4317c370cc636ec2db762bbbf91cc7b46e50be Mon Sep 17 00:00:00 2001 From: Narendra Date: Thu, 13 Aug 2026 19:51:09 +0530 Subject: [PATCH 1/4] Adds robots.txt --- docs/source/conf.py | 6 +++--- docs/source/robots.txt | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 docs/source/robots.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index c8027fc71bd54..bb8aa84ab0d5d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -89,8 +89,8 @@ html_theme_options = { "logo": { - "image_light": "_static/images/original.svg", - "image_dark": "_static/images/original_dark.svg", + "image_light": "_static/images/original.svg", + "image_dark": "_static/images/original_dark.svg", }, "use_edit_page_button": True, "navbar_center": [], @@ -111,7 +111,7 @@ # Copy agent-facing files (llms.txt) verbatim to the site root so they # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. -html_extra_path = ["llms.txt"] +html_extra_path = ["llms.txt" , "robots.txt"] html_logo = "_static/images/2x_bgwhite_original.png" diff --git a/docs/source/robots.txt b/docs/source/robots.txt new file mode 100644 index 0000000000000..c2a49f4fb82f1 --- /dev/null +++ b/docs/source/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Allow: / From fe742217c4ed2d790dfe1bae8597a6bb9fc9bf15 Mon Sep 17 00:00:00 2001 From: Narendra Date: Thu, 13 Aug 2026 20:05:38 +0530 Subject: [PATCH 2/4] Added License and removed comma space --- docs/source/conf.py | 2 +- docs/source/robots.txt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index bb8aa84ab0d5d..dfcd9fd1da464 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -111,7 +111,7 @@ # Copy agent-facing files (llms.txt) verbatim to the site root so they # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. -html_extra_path = ["llms.txt" , "robots.txt"] +html_extra_path = ["llms.txt", "robots.txt"] html_logo = "_static/images/2x_bgwhite_original.png" diff --git a/docs/source/robots.txt b/docs/source/robots.txt index c2a49f4fb82f1..57bf3b1db5dc8 100644 --- a/docs/source/robots.txt +++ b/docs/source/robots.txt @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + User-agent: * Allow: / From 434cb651015e577a8b2979d0fbaf5ebf5b3a69dc Mon Sep 17 00:00:00 2001 From: Narendra Date: Fri, 14 Aug 2026 19:08:11 +0530 Subject: [PATCH 3/4] Adding Sitemap to robots.txt --- docs/source/robots.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/robots.txt b/docs/source/robots.txt index 57bf3b1db5dc8..6466cc1a6c395 100644 --- a/docs/source/robots.txt +++ b/docs/source/robots.txt @@ -17,3 +17,5 @@ User-agent: * Allow: / + +Sitemap: https://datafusion.apache.org/sitemap.xml From aa60586c8447c297d9b1d23969a86e4f82296722 Mon Sep 17 00:00:00 2001 From: Narendra Date: Sun, 16 Aug 2026 13:28:32 +0530 Subject: [PATCH 4/4] remove license from robots.txt --- dev/release/rat_exclude_files.txt | 1 + docs/source/robots.txt | 17 ----------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 77da7db87e409..74e4279a6ab83 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -61,6 +61,7 @@ datafusion/proto-common/src/generated/prost.rs .github/ISSUE_TEMPLATE/feature_request.yml .github/workflows/docs.yaml docs/source/llms.txt +docs/source/robots.txt **/node_modules/* datafusion/wasmtest/pkg/* clippy.toml diff --git a/docs/source/robots.txt b/docs/source/robots.txt index 6466cc1a6c395..9c80d8219c4c9 100644 --- a/docs/source/robots.txt +++ b/docs/source/robots.txt @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - User-agent: * Allow: /